From 3e4572196ffd32cd4326f07428c5225fd35d6f4a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:29:33 -0800 Subject: [PATCH 01/63] test(testkit): let create_provider set provider config create_provider and install_provider return a builder that creates the provider when awaited, so tests can chain .config(key, value) for provider create --config. Existing callers are unchanged. Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 3 +- tests/testkit/testkit/src/lib.rs | 1 + tests/testkit/testkit/src/provider.rs | 140 +++++++++++++++++++------- 3 files changed, 109 insertions(+), 35 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 9cc6120620..b136777dc3 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -115,7 +115,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a provider, each with cleanup. `cx.install_provider(name, profile, - credentials)` does both. + credentials)` does both. Chain `.config(key, value)` before the `.await` + to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7c233ce834..5772ac52fd 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -58,6 +58,7 @@ pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; +pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index b68b20989c..3cd6a89114 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::future::{Future, IntoFuture}; +use std::pin::Pin; + use serde::Deserialize; use crate::context::Context; @@ -31,57 +34,126 @@ impl Context { } /// Create a provider called `name` of `provider_type` with `credentials`, - /// as pairs of environment variable and value. Cleanup deletes the - /// provider after the test's sandboxes. + /// as pairs of environment variable and value, when awaited. Cleanup + /// deletes the provider after the test's sandboxes. /// /// The CLI reads the credential values from its environment, so they - /// aren't in the reported command. - pub async fn create_provider( - &self, - name: &str, - provider_type: &str, - credentials: &[(&str, &str)], - ) -> Result { + /// aren't in the reported command. [`ProviderBuilder`] sets other + /// `provider create` options before the await. + pub fn create_provider<'a>( + &'a self, + name: &'a str, + provider_type: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Named(provider_type), credentials) + } + + /// Import `profile` with [`Context::import_profile`] and create a provider + /// called `name` of its type with [`Context::create_provider`], when + /// awaited. + pub fn install_provider<'a>( + &'a self, + name: &'a str, + profile: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Profile(profile), credentials) + } + + fn on_cleanup_delete(&self, kind: &'static str, name: &str) { + let cli = self.cli_handle(); + let owned = name.to_string(); + self.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } +} + +/// Where a [`ProviderBuilder`] gets its provider type. +enum ProviderType<'a> { + /// A profile already on the gateway. + Named(&'a str), + /// A profile, in YAML, to import first. + Profile(&'a str), +} + +/// A provider to create, from [`Context::create_provider`] or +/// [`Context::install_provider`]. Awaiting it creates the provider. +/// +/// ```ignore +/// cx.create_provider(&name, &provider_type, &[]) +/// .config("project", "production") +/// .await?; +/// ``` +#[must_use = "a ProviderBuilder does nothing until awaited"] +pub struct ProviderBuilder<'a> { + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + config: Vec, +} + +impl<'a> ProviderBuilder<'a> { + fn new( + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + ) -> Self { + Self { + cx, + name, + provider_type, + credentials, + config: Vec::new(), + } + } + + /// Set the provider config `key` to `value` (`--config key=value`). + pub fn config(mut self, key: &str, value: &str) -> Self { + self.config.push(format!("{key}={value}")); + self + } + + async fn create(self) -> Result { + let provider_type = match self.provider_type { + ProviderType::Named(provider_type) => provider_type.to_string(), + ProviderType::Profile(profile) => self.cx.import_profile(profile).await?, + }; // Deleting a missing provider succeeds, so register the delete before // the create. - self.on_cleanup_delete("provider", name); + self.cx.on_cleanup_delete("provider", self.name); let mut args = vec![ "provider", "create", "--name", - name, + self.name, "--type", - provider_type, + &provider_type, ]; - for (key, _) in credentials { + for (key, _) in self.credentials { args.extend(["--credential", key]); } - self.cli_with_env(credentials, args) + for setting in &self.config { + args.extend(["--config", setting]); + } + self.cx + .cli_with_env(self.credentials, args) .await? .success() - .context(format!("create provider {name}"))?; + .context(format!("create provider {}", self.name))?; Ok(()) } +} - /// Import `profile` with [`Context::import_profile`] and create a provider - /// called `name` of its type with [`Context::create_provider`]. - pub async fn install_provider( - &self, - name: &str, - profile: &str, - credentials: &[(&str, &str)], - ) -> Result { - let provider_type = self.import_profile(profile).await?; - self.create_provider(name, &provider_type, credentials) - .await - } +impl<'a> IntoFuture for ProviderBuilder<'a> { + type Output = Result; + type IntoFuture = Pin + 'a>>; - fn on_cleanup_delete(&self, kind: &'static str, name: &str) { - let cli = self.cli_handle(); - let owned = name.to_string(); - self.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &owned]).await?.success()?; - Ok(()) - }); + fn into_future(self) -> Self::IntoFuture { + Box::pin(self.create()) } } From 1440a1da99b84b5853f821f2d5c0490b31b6800d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:31:38 -0800 Subject: [PATCH 02/63] test(conformance): port provider files and auto-create tests to testkit Move provider_files and provider_auto_create from e2e/rust into a providers conformance suite. The auto-create test imports a run-unique profile instead of relying on a globally imported claude-code profile, so it no longer changes shared gateway state. Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 5 - e2e/rust/tests/provider_auto_create.rs | 197 --------------------- e2e/rust/tests/provider_files.rs | 176 ------------------ tests/suites/conformance/README.md | 1 + tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/providers.rs | 206 ++++++++++++++++++++++ tests/tmachine/artifacts.nix | 3 - 7 files changed, 208 insertions(+), 381 deletions(-) delete mode 100644 e2e/rust/tests/provider_auto_create.rs delete mode 100644 e2e/rust/tests/provider_files.rs create mode 100644 tests/suites/conformance/src/providers.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index c35f4ffea6..a70df6d3c4 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -61,11 +61,6 @@ name = "policy_activation" path = "tests/policy_activation.rs" required-features = ["e2e-docker"] -[[test]] -name = "provider_files" -path = "tests/provider_files.rs" -required-features = ["e2e-docker"] - [[test]] name = "oidc_pkce" path = "tests/oidc_pkce.rs" diff --git a/e2e/rust/tests/provider_auto_create.rs b/e2e/rust/tests/provider_auto_create.rs deleted file mode 100644 index 2c8828deaf..0000000000 --- a/e2e/rust/tests/provider_auto_create.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: `--provider ` auto-creates a provider from local credentials. -//! -//! When `--provider claude-code` is passed and no provider named "claude-code" exists, -//! the CLI should discover `ANTHROPIC_API_KEY` from the local environment, -//! auto-create a provider, and inject a supervisor-managed placeholder into the -//! sandbox child process environment. -//! -//! The sandbox command (`printenv ANTHROPIC_API_KEY`) verifies that the -//! placeholder made it all the way through to the sandbox process environment. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::process::Stdio; -use std::sync::Mutex; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -#[cfg(feature = "e2e-local-container-driver")] -use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; - -const TEST_API_KEY: &str = "sk-e2e-auto-provider-test-key"; -static CLAUDE_PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -fn contains_placeholder_for_env_key(output: &str, key: &str) -> bool { - let legacy = format!("openshell:resolve:env:{key}"); - let revision_prefix = "openshell:resolve:env:v"; - let revision_suffix = format!("_{key}"); - output.split_whitespace().any(|token| { - token == legacy || (token.starts_with(revision_prefix) && token.ends_with(&revision_suffix)) - }) -} - -/// Helper: delete a provider by name, ignoring errors. -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// Helper: check whether a provider already exists. -async fn provider_exists(name: &str) -> bool { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("get") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - cmd.status().await.is_ok_and(|status| status.success()) -} - -/// Helper: delete a sandbox by name, ignoring errors. -async fn delete_sandbox(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("sandbox") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// `--provider claude-code --auto-providers` with `ANTHROPIC_API_KEY` set should -/// auto-create a "claude-code" provider and inject a placeholder into the sandbox. -#[tokio::test] -async fn auto_created_provider_credential_available_in_sandbox() { - let _provider_lock = CLAUDE_PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - if provider_exists("claude-code").await { - eprintln!("Skipping test: existing provider 'claude-code' would make shared state unsafe"); - return; - } - - // Clean up any leftover from a previous run. - delete_provider("claude-code").await; - - // This test only reads the injected environment placeholder. Do not inherit - // the published image's network rules, which may be incompatible with the - // attached credential provider's startup validation. - let policy = tempfile::NamedTempFile::new().expect("create provider test policy"); - std::fs::write( - policy.path(), - r"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /etc, /proc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: {} -", - ) - .expect("write provider test policy"); - - // Create a sandbox that prints the ANTHROPIC_API_KEY env var. - // --auto-providers skips the interactive prompt. - let mut cmd = openshell_cmd(); - cmd.arg("sandbox").arg("create"); - #[cfg(feature = "e2e-local-container-driver")] - cmd.arg("--from").arg(E2E_WORKLOAD_IMAGE); - cmd.arg("--detach") - .arg("--policy") - .arg(policy.path()) - .arg("--provider") - .arg("claude-code") - .arg("--auto-providers") - .args(["--", "sh", "-c", "exec sleep infinity"]) - .env("ANTHROPIC_API_KEY", TEST_API_KEY) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .expect("failed to spawn openshell sandbox create"); - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let clean = strip_ansi(&combined); - - // Parse sandbox name for cleanup. - let sandbox_name = extract_field(&combined, "Created sandbox"); - let exec_output = if let Some(ref name) = sandbox_name { - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - name, - "--no-tty", - "--", - "printenv", - "ANTHROPIC_API_KEY", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - Some(exec_cmd.output().await.expect("failed to run sandbox exec")) - } else { - None - }; - - // Always clean up, even if assertions fail. - if let Some(ref name) = sandbox_name { - delete_sandbox(name).await; - } - delete_provider("claude-code").await; - - // Now assert. - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{clean}", - output.status.code() - ); - - let exec_output = exec_output.expect("sandbox name should be present"); - let exec_clean = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&exec_output.stdout), - String::from_utf8_lossy(&exec_output.stderr) - )); - assert!( - exec_output.status.success(), - "sandbox exec should succeed:\n{exec_clean}" - ); - - assert!( - clean.contains("Created provider claude-code"), - "output should confirm provider auto-creation:\n{clean}" - ); - - assert!( - contains_placeholder_for_env_key(&exec_clean, "ANTHROPIC_API_KEY"), - "sandbox should have placeholder ANTHROPIC_API_KEY in its environment:\n{exec_clean}" - ); - - assert!( - !exec_clean.contains(TEST_API_KEY), - "sandbox should not expose the raw ANTHROPIC_API_KEY secret:\n{exec_clean}" - ); -} diff --git a/e2e/rust/tests/provider_files.rs b/e2e/rust/tests/provider_files.rs deleted file mode 100644 index 18db5db9b7..0000000000 --- a/e2e/rust/tests/provider_files.rs +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! A provider profile serves read-only config on each workload open. - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_bin; -use openshell_e2e::harness::cli::run_cli; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::sleep; - -struct ProviderGuard { - profile: String, - provider: String, -} - -impl Drop for ProviderGuard { - fn drop(&mut self) { - let binary = openshell_bin(); - for _ in 0..20 { - let deleted = std::process::Command::new(&binary) - .args(["provider", "delete", &self.provider]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - if deleted.is_ok_and(|status| status.success()) { - break; - } - std::thread::sleep(Duration::from_millis(250)); - } - let _ = std::process::Command::new(&binary) - .args(["profile", "delete", &self.profile]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn cli_ok(args: &[&str]) -> Result<(), String> { - let (output, code) = run_cli(args).await; - if code == 0 { - Ok(()) - } else { - Err(format!( - "{} failed (exit {code}):\n{output}", - args.join(" ") - )) - } -} - -#[tokio::test] -async fn provider_file_open_update_and_detach() -> Result<(), String> { - let suffix = format!("{}-{:08x}", std::process::id(), rand::random::()); - let profile_id = format!("pf-{suffix}"); - let provider = format!("pf-{suffix}"); - let profile_yaml = include_str!("../../../examples/provider-managed-files/acme-config.yaml") - .replace("id: acme-config", &format!("id: {profile_id}")); - let mut profile_file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| error.to_string())?; - profile_file - .write_all(profile_yaml.as_bytes()) - .map_err(|error| error.to_string())?; - cli_ok(&[ - "profile", - "import", - "--file", - profile_file - .path() - .to_str() - .ok_or("profile path is not UTF-8")?, - ]) - .await?; - let _provider_guard = ProviderGuard { - profile: profile_id.clone(), - provider: provider.clone(), - }; - cli_ok(&[ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--config", - "endpoint=https://api.acme.example", - "--config", - "project=production", - ]) - .await?; - - let mut sandbox = SandboxGuard::create(&["--provider", &provider, "--no-tty"]).await?; - let path = format!("/run/openshell/providers/{provider}/client.toml"); - let environment_path = sandbox.exec(&["printenv", "ACME_CONFIG_FILE"]).await?; - if !environment_path.contains(&path) { - return Err(format!( - "provider path environment variable missing: {environment_path}" - )); - } - let before = sandbox.exec(&["cat", &path]).await?; - if !before.contains("project = \"production\"") { - return Err(format!("initial provider file content missing:\n{before}")); - } - let permissions = sandbox - .exec(&[ - "sh", - "-c", - &format!( - "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ - test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ - if (printf x >&3) 2>/dev/null; then exit 1; fi; \ - echo provider-file-permissions-ok" - ), - ]) - .await?; - if !permissions.contains("provider-file-permissions-ok") { - return Err(format!( - "provider file permission assertion did not complete:\n{permissions}" - )); - } - - cli_ok(&[ - "provider", - "update", - &provider, - "--config", - "project=staging", - "--wait", - "--timeout", - "90", - ]) - .await?; - let after = sandbox.exec(&["cat", &path]).await?; - if !after.contains("project = \"staging\"") { - return Err(format!("updated provider file content missing:\n{after}")); - } - - cli_ok(&[ - "sandbox", - "provider", - "detach", - &sandbox.name, - &provider, - "--wait", - "--timeout", - "90", - ]) - .await?; - let (detached, code) = run_cli(&[ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - &path, - ]) - .await; - if code == 0 || !detached.contains("No such file or directory") { - return Err(format!( - "detached provider path remained readable: {detached}" - )); - } - sandbox.cleanup().await; - // Let the asynchronous sandbox deletion release the provider before - // ProviderGuard removes the provider record and its profile. - sleep(Duration::from_millis(250)).await; - Ok(()) -} diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 0b19227aa9..27eadb7c99 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -19,6 +19,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `create-upload` | `sandbox create --upload`: directory basename and nested files, multiple uploads, a single file | | `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `providers` | Provider profile files: path, mode, config update, detach; `sandbox create --auto-providers` creates a provider from local credentials and injects a placeholder | | `image` | A sandbox runs the default image or the one it is created from | | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 8ac23f54bc..c0aa355e5a 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -21,6 +21,7 @@ mod lifecycle; mod persistent_workspace; mod policy_advisor; mod process; +mod providers; mod settings; mod smoke; mod ssh; diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs new file mode 100644 index 0000000000..e67a349e3a --- /dev/null +++ b/tests/suites/conformance/src/providers.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Providers attached to a sandbox: files a provider profile serves, and +//! providers that `sandbox create --provider` creates from local credentials. + +use testkit::{Context, ResultExt as _, YamlFile, policy}; + +/// The example profile that serves `client.toml` from the provider's config. +const FILES_PROFILE: &str = + include_str!("../../../../examples/provider-managed-files/acme-config.yaml"); +const AUTO_CREATE_ENV: &str = "E2E_AUTO_PROVIDER_KEY"; +const AUTO_CREATE_SECRET: &str = "sk-e2e-auto-provider-test-key"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; + +/// A provider profile's file is readable at the path its environment variable +/// names, with mode 600 and no write access. Updating the provider's config +/// updates the file, and detaching the provider removes it. +#[testkit::test] +async fn provider_file_open_update_and_detach(cx: &mut Context) -> testkit::Result { + let provider = cx.name("pf"); + let profile = FILES_PROFILE.replace("id: acme-config", &format!("id: {provider}")); + cx.install_provider(&provider, &profile, &[]) + .config("endpoint", "https://api.acme.example") + .config("project", "production") + .await?; + + let sandbox = cx + .sandbox("pf") + .spec(|spec| spec.providers = vec![provider.clone()]) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let path = format!("/run/openshell/providers/{provider}/client.toml"); + sandbox + .exec(["printenv", "ACME_CONFIG_FILE"]) + .await? + .expect_stdout(&format!("{path}\n")) + .context("the provider file's environment variable")?; + expect_project(&sandbox, &path, "production").await?; + sandbox + .exec([ + "sh", + "-c", + &format!( + "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ + test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ + if (printf x >&3) 2>/dev/null; then exit 1; fi" + ), + ]) + .await? + .success() + .context("the provider file is a read-only regular file with mode 600")?; + + cx.cli([ + "provider", + "update", + &provider, + "--config", + "project=staging", + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + expect_project(&sandbox, &path, "staging").await?; + + cx.cli([ + "sandbox", + "provider", + "detach", + sandbox.name(), + &provider, + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + let detached = sandbox.exec(["cat", &path]).await?.failure()?; + testkit::ensure!( + detached.stderr.contains("No such file or directory"), + "the detached provider's file is still there: {}", + detached.stderr + ); + Ok(()) +} + +/// `sandbox create --provider --auto-providers` creates a +/// provider named after a profile from the credential in the CLI's +/// environment. The sandbox gets a placeholder for the credential, not the +/// secret. +#[testkit::test] +async fn auto_created_provider_credential_available_in_sandbox( + cx: &mut Context, +) -> testkit::Result { + let provider = cx + .import_profile(&auto_create_profile(&cx.name("auto"))) + .await?; + // The CLI names the provider it creates after the profile. + let cli = cx.cli_handle(); + let name = provider.clone(); + cx.on_cleanup(format!("delete provider {provider}"), async move || { + cli.run(["provider", "delete", &name]).await?.success()?; + Ok(()) + }); + + // The test only reads the environment. A policy without network rules + // keeps the default policy's rules from conflicting with the provider's. + let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; + let sandbox_name = cx.name("ac"); + cx.track(&sandbox_name); + let create = cx + .cli_with_env( + &[(AUTO_CREATE_ENV, AUTO_CREATE_SECRET)], + [ + "sandbox", + "create", + "--name", + &sandbox_name, + "--no-tty", + "--detach", + "--policy", + policy.path()?, + "--provider", + &provider, + "--auto-providers", + "--", + "sh", + "-c", + "exec sleep infinity", + ], + ) + .await? + .success()?; + testkit::ensure!( + create + .report_text() + .contains(&format!("Created provider {provider}")), + "sandbox create did not report creating provider {provider}:\n{}{}", + create.stdout, + create.stderr + ); + + let sandbox = cx.adopt(&sandbox_name).await?; + sandbox.wait_ready().await?; + let output = sandbox + .exec(["printenv", AUTO_CREATE_ENV]) + .await? + .success()?; + let value = output.stdout.trim(); + testkit::ensure!( + !value.contains(AUTO_CREATE_SECRET), + "the sandbox has the raw credential in {AUTO_CREATE_ENV}" + ); + testkit::ensure!( + is_placeholder_for(value, AUTO_CREATE_ENV), + "expected a placeholder for {AUTO_CREATE_ENV} in the sandbox; got {value:?}" + ); + Ok(()) +} + +async fn expect_project(sandbox: &testkit::Sandbox, path: &str, project: &str) -> testkit::Result { + let content = sandbox.exec(["cat", path]).await?.success()?.stdout; + testkit::ensure!( + content.contains(&format!("project = \"{project}\"")), + "expected project {project} in the provider file:\n{content}" + ); + Ok(()) +} + +/// A profile `id` whose credential the CLI discovers in [`AUTO_CREATE_ENV`]. +/// Its endpoint makes the gateway inject the credential without a policy +/// binding; nothing connects to it. +fn auto_create_profile(id: &str) -> String { + format!( + "id: {id} +display_name: E2E Auto-created Provider +category: other +credentials: + - name: api_key + env_vars: [{AUTO_CREATE_ENV}] + required: true + auth_style: header + header_name: x-api-key +discovery: + credentials: [api_key] +endpoints: + - host: api.auto-provider.example + port: 443 + protocol: rest + access: read-write + enforcement: enforce +binaries: [/usr/bin/curl] +" + ) +} + +/// Whether `value` is the placeholder for the credential in `key`, in the +/// unversioned or the revisioned form. +fn is_placeholder_for(value: &str, key: &str) -> bool { + value == format!("{PLACEHOLDER_PREFIX}{key}") + || (value.starts_with(&format!("{PLACEHOLDER_PREFIX}v")) + && value.ends_with(&format!("_{key}"))) +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index 764889a1ee..aee901d808 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -97,9 +97,6 @@ let "local_driver_token_restart" "podman_corporate_proxy" "podman_gateway_start" - # Its move to conformance is deferred until the isolation work in #4120; see - # #4246. - "provider_auto_create" # Needs the SPIFFE fixtures that the wrapper sets up. Wrapper lane. "provider_token_exchange" # One test times out reaching support-container IPs from the sandbox in the From 7d65705b9e5e9591c7f1478cafdb20eef152ba15 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:33:55 -0800 Subject: [PATCH 03/63] test(testkit): add service_get for exposed sandbox services Requests go to the gateway's endpoint with the service URL's host as the Host header, which is how the gateway routes service traffic. Over TLS the client trusts the gateway CA and presents the registered gateway's client certificate, through openshell-crypto's rustls configuration. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 3 + tests/suites/drivers/podman/Cargo.lock | 3 + tests/suites/features/Cargo.lock | 3 + tests/support/Cargo.lock | 3 + tests/testkit/Cargo.lock | 3 + tests/testkit/README.md | 4 + tests/testkit/testkit/Cargo.toml | 5 + tests/testkit/testkit/src/error.rs | 6 + tests/testkit/testkit/src/gateway.rs | 6 + tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/service.rs | 190 +++++++++++++++++++++++++ 11 files changed, 228 insertions(+) create mode 100644 tests/testkit/testkit/src/service.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index cf08ce94fe..3768b6b33c 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2405,11 +2405,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 459d6889c4..f9615aec41 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2393,11 +2393,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4c65ad4021..2d2bc6cf7d 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2477,11 +2477,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index bc18df94cd..65259703e7 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2411,11 +2411,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 7e808b028d..17fc9196f3 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2384,11 +2384,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index b136777dc3..f8aef833f0 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -118,6 +118,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. Chain `.config(key, value)` before the `.await` to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. +- `cx.service_get(service_url, headers)` to request an exposed sandbox + service. It connects to the gateway's endpoint and sends the service URL's + host as the Host header, so the test machine doesn't need to resolve + service hosts. It returns the status and body whatever the status. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 7b8ebd90ab..464d5b0f5b 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -17,6 +17,7 @@ name = "testkit" [dependencies] openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-policy = { path = "../../../crates/openshell-policy" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } @@ -26,6 +27,10 @@ libtest-mimic = "0.8" linkme = "0.3" quick-junit = "0.8" rand = "0.9" +# Service requests use a rustls configuration from openshell-crypto, so +# reqwest brings no crypto provider or root store of its own. +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider"] } +rustls = { version = "0.23", default-features = false, features = ["std"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 18ce6138c7..574979c1b8 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -99,6 +99,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::from_error(&error) + } +} + impl From for Error { fn from(error: std::io::Error) -> Self { Self::from_error(&error) diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index fef05833b7..79bd5fe827 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -121,6 +121,12 @@ impl Gateway { }) } + /// The directory holding the gateway's CA and client certificate, for + /// an mTLS gateway. + pub(crate) fn mtls_dir(&self) -> Option<&Path> { + self.mtls_dir.as_deref() + } + /// One line naming the gateway, for test logs. pub fn describe(&self) -> String { let auth = match (&self.mtls_dir, self.endpoint.starts_with("https://")) { diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 5772ac52fd..ea005d6a4e 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -42,6 +42,7 @@ mod provider; mod ready; mod registry; mod run; +pub mod service; mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; @@ -62,6 +63,7 @@ pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; +pub use service::{SERVICE_REQUEST_TIMEOUT, ServiceResponse}; pub use testkit_macros::test; pub use tonic::Code as GrpcCode; pub use yaml_file::YamlFile; diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs new file mode 100644 index 0000000000..f914554bf3 --- /dev/null +++ b/tests/testkit/testkit/src/service.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Requests to sandbox services that the gateway exposes. +//! +//! A service URL names the sandbox and service in its host, such as +//! `http://default--my-sandbox.openshell.localhost:8080/`. The gateway routes +//! service requests by their Host header on its own listener, so testkit +//! connects to the gateway's endpoint and sends the service URL's host as the +//! Host header. The machine that runs the tests doesn't need to resolve +//! service hosts. +//! +//! The request keeps the service URL's scheme. A gateway listening on +//! loopback reports `http` service URLs and accepts plaintext service +//! requests on its TLS listener. Over TLS, testkit verifies the gateway's +//! certificate against the endpoint's host and presents the registered +//! gateway's client certificate. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::Url; +use reqwest::header::HOST; +use rustls::RootCertStore; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +use crate::context::Context; +use crate::error::{Error, Result, ResultExt as _}; +use crate::gateway::Gateway; + +/// How long one service request may take. +pub const SERVICE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// The response to a service request. +#[derive(Debug)] +pub struct ServiceResponse { + pub status: u16, + pub body: String, +} + +impl Context { + /// Send `GET service_url` with `headers` through the test's gateway and + /// return the response, whatever its status. See [the module + /// docs](crate::service) for how the request reaches the gateway. + pub async fn service_get( + &self, + service_url: &str, + headers: &[(&str, &str)], + ) -> Result { + let cli = self.cli_handle(); + let request = ServiceRequest::new(cli.gateway(), service_url)?; + let mut builder = client(cli.gateway())? + .get(request.target) + .header(HOST, &request.host); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let what = format!("GET {service_url} through {}", cli.gateway().endpoint); + let response = builder.send().await.context(&what)?; + let status = response.status().as_u16(); + let body = response.text().await.context(&what)?; + Ok(ServiceResponse { status, body }) + } +} + +/// Where a service request goes, and the Host header that selects the +/// service. +#[derive(Debug, PartialEq, Eq)] +struct ServiceRequest { + target: Url, + host: String, +} + +impl ServiceRequest { + fn new(gateway: &Gateway, service_url: &str) -> Result { + let service = parse(service_url, "service URL")?; + let endpoint = parse(&gateway.endpoint, "gateway endpoint")?; + let service_host = service + .host_str() + .ok_or_else(|| Error::msg(format!("service URL {service_url} has no host")))?; + let host = match service.port() { + Some(port) => format!("{service_host}:{port}"), + None => service_host.to_string(), + }; + let mut target = service.clone(); + target + .set_host(endpoint.host_str()) + .map_err(Error::from_display) + .context(format!("send {service_url} to {}", gateway.endpoint))?; + target + .set_port(endpoint.port_or_known_default()) + .map_err(|()| Error::msg(format!("send {service_url} to {}", gateway.endpoint)))?; + Ok(Self { target, host }) + } +} + +fn parse(url: &str, what: &str) -> Result { + Url::parse(url) + .map_err(Error::from_display) + .context(format!("parse {what} {url}")) +} + +/// An HTTP client that trusts the gateway's CA and presents its client +/// certificate. Without a registered mTLS gateway it trusts no certificate, +/// so only plaintext requests succeed. +fn client(gateway: &Gateway) -> Result { + let builder = openshell_crypto::tls::client_builder(); + let tls = match gateway.mtls_dir() { + Some(dir) => { + let mut roots = RootCertStore::empty(); + for certificate in certificates(&dir.join("ca.crt"))? { + roots + .add(certificate) + .map_err(Error::from_display) + .context("trust the gateway CA")?; + } + builder + .with_root_certificates(roots) + .with_client_auth_cert( + certificates(&dir.join("tls.crt"))?, + private_key(&dir.join("tls.key"))?, + ) + .map_err(Error::from_display) + .context("use the gateway client certificate")? + } + None => builder + .with_root_certificates(RootCertStore::empty()) + .with_no_client_auth(), + }; + reqwest::Client::builder() + .use_preconfigured_tls(Arc::new(tls)) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(SERVICE_REQUEST_TIMEOUT) + .build() + .context("build the service HTTP client") +} + +fn certificates(path: &Path) -> Result>> { + CertificateDer::pem_file_iter(path) + .and_then(Iterator::collect) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +fn private_key(path: &Path) -> Result> { + PrivateKeyDer::from_pem_file(path) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::ServiceRequest; + use crate::gateway::Gateway; + + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { + let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); + ServiceRequest::new(&gateway, service_url).expect("service request") + } + + /// A loopback gateway with TLS reports `http` service URLs; the request + /// goes to the gateway's address in plaintext, with the service as Host. + #[test] + fn plaintext_service_on_a_tls_gateway() { + let request = request( + "https://127.0.0.1:17670", + "http://default--sb.openshell.localhost:17670/path?q=1", + ); + assert_eq!(request.target.as_str(), "http://127.0.0.1:17670/path?q=1"); + assert_eq!(request.host, "default--sb.openshell.localhost:17670"); + } + + /// The request goes to the gateway's port even when the service URL + /// names another one, such as the gateway's port inside a cluster. + #[test] + fn service_port_differs_from_the_endpoint() { + let request = request( + "https://openshell.openshell.svc.cluster.local:8443", + "https://default--sb--web.openshell.localhost/", + ); + assert_eq!( + request.target.as_str(), + "https://openshell.openshell.svc.cluster.local:8443/" + ); + assert_eq!(request.host, "default--sb--web.openshell.localhost"); + } +} From 2ab1ef4cfeb840ff3d7cd239eb05ad13aaa70688 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:37:08 -0800 Subject: [PATCH 04/63] test(features): port service bearer passthrough to a service-exposure suite Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 5 + TESTING.md | 6 + e2e/rust/Cargo.lock | 2 - e2e/rust/Cargo.toml | 7 - e2e/rust/tests/service_bearer_passthrough.rs | 350 ------------------ tests/suites/features/Cargo.lock | 8 + tests/suites/features/Cargo.toml | 2 +- .../features/service-exposure/Cargo.toml | 16 + .../tests/service_exposure.rs | 119 ++++++ 9 files changed, 155 insertions(+), 360 deletions(-) delete mode 100644 e2e/rust/tests/service_bearer_passthrough.rs create mode 100644 tests/suites/features/service-exposure/Cargo.toml create mode 100644 tests/suites/features/service-exposure/tests/service_exposure.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 86073d5b56..1bed77dced 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -105,6 +105,11 @@ jobs: {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 5a8a3942aa..91de7ca39c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -656,6 +656,12 @@ supervises its `ssh` process: it reconnects after losing the SSH transport and forwards a SIGTERM to `ssh`. The tests find those processes in `/proc`, so they need a Linux test host. CI runs them on Docker rootful. +The `service-exposure` feature suite (`tests/suites/features/service-exposure`) +exposes a service in a sandbox and checks how the gateway forwards the client's +`Authorization` header. The test sends requests to the gateway's endpoint with +the service URL's host as the Host header, so the test machine doesn't resolve +service hosts. CI runs it on the same environments as `host-gateway`. + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 287d2c65ec..38458f7de8 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2204,7 +2204,6 @@ dependencies = [ "opentelemetry-proto", "prost", "rand 0.9.5", - "rustls", "serde", "serde_json", "serial_test", @@ -2212,7 +2211,6 @@ dependencies = [ "sha2 0.10.9", "tempfile", "tokio", - "tokio-rustls", "tokio-stream", "tonic", "tonic-prost", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index a70df6d3c4..13ba00064d 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -71,11 +71,6 @@ name = "vm_overlay" path = "tests/vm_overlay.rs" required-features = ["e2e-vm"] -[[test]] -name = "service_bearer_passthrough" -path = "tests/service_bearer_passthrough.rs" -required-features = ["e2e-docker"] - [[test]] name = "rootfs_tar" path = "tests/rootfs_tar.rs" @@ -271,13 +266,11 @@ sha1 = "0.10" sha2 = "0.10" hex = "0.4" rand = "0.9" -rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tonic = { version = "0.14", features = ["transport"] } tonic-prost = "0.14" -tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tower = "0.5" url = "2" diff --git a/e2e/rust/tests/service_bearer_passthrough.rs b/e2e/rust/tests/service_bearer_passthrough.rs deleted file mode 100644 index 75415c7df2..0000000000 --- a/e2e/rust/tests/service_bearer_passthrough.rs +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! Verifies application bearer authorization behavior through an exposed -//! `OpenShell` service. - -use std::fs::File; -use std::io::BufReader; -use std::net::Ipv4Addr; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use http_body_util::{BodyExt as _, Empty}; -use hyper::client::conn::http1; -use hyper::{Request, StatusCode, header}; -use hyper_util::rt::TokioIo; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard}; -use rustls::RootCertStore; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; -use serde_json::Value; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; -use tokio::time::{sleep, timeout}; -use tokio_rustls::TlsConnector; -use url::Position; - -const SERVICE_PORT: &str = "4500"; -const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; -const READY_TIMEOUT: Duration = Duration::from_mins(1); -const HEADER_ECHO_SERVER: &str = r#" -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - body = self.headers.get("Authorization", "").encode() - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, _format, *_args): - pass - -ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() -"#; - -async fn run_cli(args: &[&str]) -> Result { - let mut command = openshell_cmd(); - command - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - command - .output() - .await - .map_err(|error| format!("failed to run openshell: {error}")) -} - -enum ServiceTransport { - Http, - Https { - connector: TlsConnector, - server_name: ServerName<'static>, - }, -} - -struct ServiceTarget { - port: u16, - authority: String, - path: String, - transport: ServiceTransport, -} - -impl ServiceTarget { - fn from_url(url: &str) -> Result { - let url = url::Url::parse(url).map_err(|error| format!("invalid service URL: {error}"))?; - let host = url - .host_str() - .ok_or_else(|| "service URL omitted its host".to_string())?; - let port = url - .port_or_known_default() - .ok_or_else(|| "service URL omitted its port".to_string())?; - let transport = match url.scheme() { - "http" => ServiceTransport::Http, - "https" => ServiceTransport::Https { - connector: e2e_tls_connector()?, - server_name: ServerName::try_from(host.to_string()) - .map_err(|error| format!("invalid service TLS server name: {error}"))?, - }, - scheme => return Err(format!("unsupported service URL scheme {scheme:?}")), - }; - let authority = url[Position::BeforeHost..Position::AfterPort].to_string(); - let path = url.query().map_or_else( - || url.path().to_string(), - |query| format!("{}?{query}", url.path()), - ); - - Ok(Self { - port, - authority, - path, - transport, - }) - } -} - -fn e2e_mtls_dir() -> Result { - let config_home = std::env::var_os("XDG_CONFIG_HOME") - .ok_or_else(|| "XDG_CONFIG_HOME is required for an HTTPS service URL".to_string())?; - let gateway = std::env::var("OPENSHELL_GATEWAY") - .map_err(|_| "OPENSHELL_GATEWAY is required for an HTTPS service URL".to_string())?; - Ok(PathBuf::from(config_home) - .join("openshell/gateways") - .join(gateway) - .join("mtls")) -} - -fn load_certificates( - path: &Path, - description: &str, -) -> Result>, String> { - let file = File::open(path) - .map_err(|error| format!("open {description} '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|error| format!("parse {description} '{}': {error}", path.display())) -} - -fn load_private_key(path: &Path) -> Result, String> { - let file = File::open(path) - .map_err(|error| format!("open client TLS key '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - PrivateKeyDer::pem_reader_iter(&mut reader) - .next() - .transpose() - .map_err(|error| format!("parse client TLS key '{}': {error}", path.display()))? - .ok_or_else(|| format!("client TLS key '{}' is empty", path.display())) -} - -fn e2e_tls_connector() -> Result { - let mtls_dir = e2e_mtls_dir()?; - let ca_path = mtls_dir.join("ca.crt"); - let mut roots = RootCertStore::empty(); - for certificate in load_certificates(&ca_path, "gateway CA certificate")? { - roots.add(certificate).map_err(|error| { - format!( - "add gateway CA certificate '{}': {error}", - ca_path.display() - ) - })?; - } - let client_certificates = - load_certificates(&mtls_dir.join("tls.crt"), "client TLS certificate")?; - let client_key = load_private_key(&mtls_dir.join("tls.key"))?; - let config = openshell_crypto::tls::client_builder() - .with_root_certificates(roots) - .with_client_auth_cert(client_certificates, client_key) - .map_err(|error| format!("build e2e mTLS client configuration: {error}"))?; - Ok(TlsConnector::from(Arc::new(config))) -} - -async fn request_over_stream( - stream: S, - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) - .await - .map_err(|error| format!("start HTTP connection: {error}"))?; - tokio::spawn(async move { - let _ = connection.await; - }); - - let request = Request::builder() - .uri(&target.path) - .header(header::HOST, &target.authority) - .header(header::AUTHORIZATION, authorization) - .body(Empty::::new()) - .map_err(|error| format!("build service request: {error}"))?; - let response = sender - .send_request(request) - .await - .map_err(|error| format!("send service request: {error}"))?; - let status = response.status(); - let body = response - .into_body() - .collect() - .await - .map_err(|error| format!("read service response: {error}"))? - .to_bytes(); - let body = String::from_utf8(body.to_vec()) - .map_err(|error| format!("service returned non-UTF-8 data: {error}"))?; - Ok((status, body)) -} - -#[allow( - clippy::disallowed_methods, - reason = "the e2e workspace does not depend on openshell-core" -)] -async fn request_service( - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> { - // The service hostname is a virtual routing authority. Dial the loopback - // gateway directly so the test does not depend on the host resolver - // recognizing arbitrary subdomains of `.localhost`. - let stream = TcpStream::connect((Ipv4Addr::LOCALHOST, target.port)) - .await - .map_err(|error| format!("connect to loopback service gateway: {error}"))?; - let _ = stream.set_nodelay(true); - match &target.transport { - ServiceTransport::Http => request_over_stream(stream, target, authorization).await, - ServiceTransport::Https { - connector, - server_name, - } => { - let stream = connector - .connect(server_name.clone(), stream) - .await - .map_err(|error| format!("start service TLS connection: {error}"))?; - request_over_stream(stream, target, authorization).await - } - } -} - -async fn wait_for_authorization(url: &str, expected: &str) -> Result<(), String> { - // Parse the URL and load TLS material before the retry loop so permanent - // test-configuration errors fail immediately instead of looking like - // service-readiness timeouts. - let target = ServiceTarget::from_url(url)?; - let mut last_observation = "no request attempted".to_string(); - let result = timeout(READY_TIMEOUT, async { - loop { - match request_service(&target, BEARER_TOKEN).await { - Ok((StatusCode::OK, body)) if body == expected => return Ok(()), - Ok((StatusCode::OK, body)) => { - return Err(format!( - "service received unexpected Authorization value: {body:?}" - )); - } - Ok((status, body)) - if matches!( - status, - StatusCode::BAD_GATEWAY - | StatusCode::PRECONDITION_FAILED - | StatusCode::SERVICE_UNAVAILABLE - ) => - { - last_observation = - format!("service returned retryable status {status} with body {body:?}"); - sleep(Duration::from_millis(250)).await; - } - Err(error) => { - last_observation = error; - sleep(Duration::from_millis(250)).await; - } - Ok((status, body)) => { - return Err(format!( - "service returned unexpected status {status} with body {body:?}" - )); - } - } - } - }) - .await; - - match result { - Ok(result) => result, - Err(_) => Err(format!( - "timed out waiting for the exposed service; last observation: {last_observation}" - )), - } -} - -#[tokio::test] -async fn service_bearer_passthrough_preserves_authorization_header() { - let sandbox_name = format!("svc-auth-{}", std::process::id()); - let create = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--from", - E2E_WORKLOAD_IMAGE, - "--expose", - SERVICE_PORT, - "--output", - "json", - "--detach", - "--no-tty", - "--", - "python3", - "-c", - HEADER_ECHO_SERVER, - ]) - .await - .expect("run sandbox create"); - assert!( - create.status.success(), - "sandbox create failed with exit {:?}: {}", - create.status.code(), - String::from_utf8_lossy(&create.stderr) - ); - let mut sandbox = SandboxGuard::manage_existing(sandbox_name.clone()); - - let created: Value = serde_json::from_slice(&create.stdout).expect("parse sandbox create JSON"); - let service_url = created - .get("service_urls") - .and_then(|urls| urls.get("")) - .and_then(Value::as_str) - .expect("unnamed service URL in create response"); - - wait_for_authorization(service_url, "") - .await - .expect("default mode should strip Authorization"); - - let expose = run_cli(&[ - "service", - "expose", - &sandbox_name, - SERVICE_PORT, - "--authorization-mode", - "bearer-passthrough", - ]) - .await - .expect("run service re-expose"); - assert!( - expose.status.success(), - "service re-expose failed with exit {:?}: {}", - expose.status.code(), - String::from_utf8_lossy(&expose.stderr) - ); - - wait_for_authorization(service_url, BEARER_TOKEN) - .await - .expect("passthrough mode should preserve Authorization"); - - sandbox.cleanup().await; -} diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 2d2bc6cf7d..a92741823a 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -3425,6 +3425,14 @@ dependencies = [ "serde", ] +[[package]] +name = "service-exposure" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde_json", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index e12a0a1a02..f605f7c85b 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "service-exposure"] diff --git a/tests/suites/features/service-exposure/Cargo.toml b/tests/suites/features/service-exposure/Cargo.toml new file mode 100644 index 0000000000..73b970a33d --- /dev/null +++ b/tests/suites/features/service-exposure/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "service-exposure" +description = "Features suite: requests reach a sandbox service the gateway exposes" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/service-exposure/tests/service_exposure.rs b/tests/suites/features/service-exposure/tests/service_exposure.rs new file mode 100644 index 0000000000..8b78e49f80 --- /dev/null +++ b/tests/suites/features/service-exposure/tests/service_exposure.rs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exposed sandbox services and the application's `Authorization` header. +//! +//! The sandbox runs a server that echoes the request's `Authorization` +//! header, and the test requests it through the gateway with +//! `cx.service_get`. + +use std::time::Duration; + +use testkit::{Context, Poll, ResultExt as _, poll_until}; + +const SERVICE_PORT: &str = "4500"; +const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; +/// How long an exposed service may take to start answering, or to apply a +/// changed authorization mode. +const SERVICE_TIMEOUT: Duration = Duration::from_mins(1); +const HEADER_ECHO_SERVER: &str = r#" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = self.headers.get("Authorization", "").encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + pass + +ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() +"#; + +/// An exposed service doesn't receive the client's `Authorization` header by +/// default. Exposing it again with `--authorization-mode bearer-passthrough` +/// forwards the header unchanged. +#[testkit::test] +async fn bearer_passthrough_preserves_authorization_header(cx: &mut Context) -> testkit::Result { + let builder = cx + .cli_sandbox("svc") + .image(testkit::python_image()) + .args(["--expose", SERVICE_PORT, "--output", "json", "--detach"]) + .command(["python3", "-c", HEADER_ECHO_SERVER]); + let name = builder.name().to_string(); + let created: serde_json::Value = builder + .run() + .await? + .success() + .context(format!("create sandbox {name}"))? + .json()?; + let sandbox = cx.adopt(&name).await?; + let service_url = created["service_urls"][""] + .as_str() + .ok_or_else(|| testkit::Error::msg(format!("no unnamed service URL in {created}")))? + .to_string(); + sandbox.wait_ready().await?; + + wait_for_authorization(cx, &service_url, "") + .await + .context("the default mode strips Authorization")?; + + cx.cli([ + "service", + "expose", + &name, + SERVICE_PORT, + "--authorization-mode", + "bearer-passthrough", + ]) + .await? + .success()?; + wait_for_authorization(cx, &service_url, BEARER_TOKEN) + .await + .context("bearer-passthrough mode preserves Authorization") +} + +/// Request `service_url` with [`BEARER_TOKEN`] until the service echoes +/// `expected`. Connection errors and the gateway's statuses for a service +/// that isn't reachable yet are retried; any other answer fails. +async fn wait_for_authorization( + cx: &Context, + service_url: &str, + expected: &str, +) -> testkit::Result { + poll_until( + &format!("{service_url} to receive Authorization {expected:?}"), + SERVICE_TIMEOUT, + Duration::from_millis(250), + async || { + let response = match cx + .service_get(service_url, &[("authorization", BEARER_TOKEN)]) + .await + { + Ok(response) => response, + Err(error) => return Ok(Poll::Pending(error.to_string())), + }; + match response.status { + 200 if response.body == expected => Ok(Poll::Ready(())), + 200 => Err(testkit::Error::msg(format!( + "the service received Authorization {:?}", + response.body + ))), + // Bad gateway, precondition failed, service unavailable. + 502 | 412 | 503 => Ok(Poll::Pending(format!( + "status {} with body {:?}", + response.status, response.body + ))), + status => Err(testkit::Error::msg(format!( + "the service answered status {status} with body {:?}", + response.body + ))), + } + }, + ) + .await +} From 2aecd4626d873c5b8e7eabfce972b027139542ce Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 05/63] fix(testkit): pass the service client's rustls config by value Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/service.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs index f914554bf3..8e2ee3102a 100644 --- a/tests/testkit/testkit/src/service.rs +++ b/tests/testkit/testkit/src/service.rs @@ -17,7 +17,6 @@ //! gateway's client certificate. use std::path::Path; -use std::sync::Arc; use std::time::Duration; use reqwest::Url; @@ -130,7 +129,7 @@ fn client(gateway: &Gateway) -> Result { .with_no_client_auth(), }; reqwest::Client::builder() - .use_preconfigured_tls(Arc::new(tls)) + .use_preconfigured_tls(tls) .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(SERVICE_REQUEST_TIMEOUT) @@ -153,9 +152,18 @@ fn private_key(path: &Path) -> Result> { #[cfg(test)] mod tests { - use super::ServiceRequest; + use super::{ServiceRequest, client}; use crate::gateway::Gateway; + /// reqwest accepts the rustls configuration only as a plain + /// `ClientConfig`; anything else fails when the client is built. + #[test] + fn client_accepts_the_tls_configuration() { + let gateway = + Gateway::from_endpoint("http://127.0.0.1:8080".to_string(), "default".to_string()); + client(&gateway).expect("service HTTP client"); + } + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); ServiceRequest::new(&gateway, service_url).expect("service request") From d3301b2439881a35c7f49a1db58aa21a854b0199 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 06/63] test(conformance): create the auto-create sandbox from the workload image Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index e67a349e3a..c5b0cdfec8 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -110,6 +110,7 @@ async fn auto_created_provider_credential_available_in_sandbox( // keeps the default policy's rules from conflicting with the provider's. let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; let sandbox_name = cx.name("ac"); + let image = testkit::python_image(); cx.track(&sandbox_name); let create = cx .cli_with_env( @@ -121,6 +122,8 @@ async fn auto_created_provider_credential_available_in_sandbox( &sandbox_name, "--no-tty", "--detach", + "--from", + &image, "--policy", policy.path()?, "--provider", From e59f8f3ed482d3d74f629b88722085628a386ede Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:21:23 -0800 Subject: [PATCH 07/63] test(testkit): add a typed provider profile builder Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 16 ++ tests/suites/drivers/podman/Cargo.lock | 16 ++ tests/support/Cargo.lock | 16 ++ tests/testkit/Cargo.lock | 17 ++ tests/testkit/README.md | 5 +- tests/testkit/testkit/Cargo.toml | 4 + tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/policy.rs | 20 +- tests/testkit/testkit/src/profile.rs | 307 +++++++++++++++++++++++++ 9 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 tests/testkit/testkit/src/profile.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 3768b6b33c..357caddb55 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2368,6 +2368,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2407,6 +2422,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index f9615aec41..e5a292df0a 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2356,6 +2356,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2395,6 +2410,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 65259703e7..e7dc4969ad 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2374,6 +2374,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2413,6 +2428,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 17fc9196f3..4dcdb5796f 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2347,6 +2347,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2386,8 +2401,10 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "reqwest", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index f8aef833f0..f6a32d86a9 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -116,7 +116,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. `id`, and `cx.create_provider(name, type, credentials)` to create a provider, each with cleanup. `cx.install_provider(name, profile, credentials)` does both. Chain `.config(key, value)` before the `.await` - to set provider config. + to set provider config. Build the profile with `testkit::Profile` and + `testkit::Credential`, whose setters are named after the profile YAML + fields; `policy::Endpoint` configures its endpoints. `profile.to_yaml()` + renders it for these calls. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.service_get(service_url, headers)` to request an exposed sandbox service. It connects to the gateway's endpoint and sends the service URL's diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 464d5b0f5b..48cd895a9f 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -19,6 +19,7 @@ openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-policy = { path = "../../../crates/openshell-policy" } +openshell-providers = { path = "../../../crates/openshell-providers" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } clap = { version = "4", features = ["derive"] } @@ -41,6 +42,9 @@ tonic = "0.14" [target.'cfg(unix)'.dependencies] libc = "0.2" +[dev-dependencies] +prost-types = "0.14" + # A test program, run by `cargo test` and nextest through `testkit::main`'s # libtest interface, that checks the program's own subcommands. [[test]] diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index ea005d6a4e..1d1b375eef 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -37,6 +37,7 @@ mod image; mod output; pub mod policy; mod poll; +pub mod profile; pub mod program; mod provider; mod ready; @@ -58,6 +59,7 @@ pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; +pub use profile::{Credential, Profile}; pub use program::main; pub use provider::ProviderBuilder; pub use ready::check as check_ready; diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..0754651a24 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -89,6 +89,10 @@ pub trait Endpoint { fn credential_binding(&mut self, provider: &str) -> &mut Self; /// `allow_uninspected_credentials: true`. fn allow_uninspected_credentials(&mut self) -> &mut Self; + /// `enforcement: enforce`. + fn enforce(&mut self) -> &mut Self; + /// `path: `. + fn path(&mut self, path: &str) -> &mut Self; } impl Endpoint for proto::NetworkEndpoint { @@ -125,6 +129,16 @@ impl Endpoint for proto::NetworkEndpoint { self.allow_uninspected_credentials = true; self } + + fn enforce(&mut self) -> &mut Self { + self.enforcement = proto::NetworkEnforcementMode::Enforce as i32; + self + } + + fn path(&mut self, path: &str) -> &mut Self { + self.path = path.to_string(); + self + } } #[cfg(test)] @@ -155,7 +169,9 @@ mod tests { &["/usr/bin/python*"], ) .websocket(Access::ReadWrite) - .allow_uninspected_credentials(); + .allow_uninspected_credentials() + .enforce() + .path("/ws"); let yaml = format!( "{BASE}network_policies: @@ -174,6 +190,8 @@ mod tests { protocol: websocket access: read-write allow_uninspected_credentials: true + enforcement: enforce + path: /ws binaries: - path: /usr/bin/python* " diff --git a/tests/testkit/testkit/src/profile.rs b/tests/testkit/testkit/src/profile.rs new file mode 100644 index 0000000000..10217ff040 --- /dev/null +++ b/tests/testkit/testkit/src/profile.rs @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Provider profiles for tests, built in Rust and rendered as the YAML that +//! [`Context::import_profile`](crate::Context::import_profile) and +//! [`Context::install_provider`](crate::Context::install_provider) take. +//! +//! ```ignore +//! use testkit::policy::{Access, Endpoint as _}; +//! +//! let profile = testkit::Profile::new(&cx.name("auto")) +//! .credential( +//! testkit::Credential::new("api_key") +//! .env_var("API_KEY") +//! .required() +//! .auth_style("header") +//! .header_name("x-api-key"), +//! ) +//! .endpoint("api.example.test", 443, |ep| { +//! ep.rest(Access::ReadWrite).enforce(); +//! }) +//! .binaries(["/usr/bin/curl"]); +//! cx.import_profile(&profile.to_yaml()?).await?; +//! ``` +//! +//! Profile endpoints are policy endpoints, so [`policy::Endpoint`] configures +//! them. Setters are named after the YAML fields they set. +//! +//! [`policy::Endpoint`]: crate::policy::Endpoint + +use openshell_providers::{ProviderTypeProfile, profile_to_yaml}; +use openshell_sdk::raw::proto; + +use crate::error::{Error, Result, ResultExt as _}; + +pub use proto::ProviderProfileCategory as Category; + +/// A provider profile. Its display name defaults to its id, and its category +/// to `other`. +#[derive(Clone, Debug)] +#[must_use] +pub struct Profile { + profile: proto::ProviderProfile, +} + +impl Profile { + /// A profile with `id`, the provider type it defines. + pub fn new(id: &str) -> Self { + Self { + profile: proto::ProviderProfile { + id: id.to_string(), + display_name: id.to_string(), + category: Category::Other as i32, + ..proto::ProviderProfile::default() + }, + } + } + + /// The profile's `id`. + pub fn id(&self) -> &str { + &self.profile.id + } + + /// `display_name`. + pub fn display_name(mut self, name: &str) -> Self { + self.profile.display_name = name.to_string(); + self + } + + /// `category`. + pub fn category(mut self, category: Category) -> Self { + self.profile.category = category as i32; + self + } + + /// Add `credential` to `credentials`. + pub fn credential(mut self, credential: Credential) -> Self { + self.profile.credentials.push(credential.credential); + self + } + + /// `discovery: {credentials: [..]}`: the credentials the CLI looks for in + /// its environment when it creates the provider itself. + pub fn discover(mut self, credentials: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.profile + .discovery + .get_or_insert_with(proto::ProviderProfileDiscovery::default) + .credentials + .extend(credentials.into_iter().map(Into::into)); + self + } + + /// Add an endpoint for `host:port` to `endpoints`, configured by + /// `configure` with [`policy::Endpoint`](crate::policy::Endpoint). + pub fn endpoint( + mut self, + host: &str, + port: u16, + configure: impl FnOnce(&mut proto::NetworkEndpoint), + ) -> Self { + let mut endpoint = proto::NetworkEndpoint { + host: host.to_string(), + port: port.into(), + ..proto::NetworkEndpoint::default() + }; + configure(&mut endpoint); + self.profile.endpoints.push(endpoint); + self + } + + /// Add `paths` to `binaries`. + pub fn binaries(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.profile.binaries.extend( + paths + .into_iter() + .map(|path| proto::NetworkBinary { path: path.into() }), + ); + self + } + + /// The profile as YAML, as `profile export` writes it. + pub fn to_yaml(&self) -> Result { + profile_to_yaml(&ProviderTypeProfile::from_proto(&self.profile)) + .map_err(Error::from_display) + .context(format!("render provider profile {}", self.profile.id)) + } +} + +/// A credential in a [`Profile`]. +#[derive(Clone, Debug)] +#[must_use] +pub struct Credential { + credential: proto::ProviderProfileCredential, +} + +impl Credential { + /// A credential called `name`. + pub fn new(name: &str) -> Self { + Self { + credential: proto::ProviderProfileCredential { + name: name.to_string(), + ..proto::ProviderProfileCredential::default() + }, + } + } + + /// Add `var` to `env_vars`. + pub fn env_var(mut self, var: &str) -> Self { + self.credential.env_vars.push(var.to_string()); + self + } + + /// `required: true`. + pub fn required(mut self) -> Self { + self.credential.required = true; + self + } + + /// `auth_style`, such as `bearer` or `header`. + pub fn auth_style(mut self, style: &str) -> Self { + self.credential.auth_style = style.to_string(); + self + } + + /// `header_name`. + pub fn header_name(mut self, name: &str) -> Self { + self.credential.header_name = name.to_string(); + self + } + + /// `refresh`, for a credential the gateway refreshes. + pub fn refresh(mut self, refresh: proto::ProviderCredentialRefresh) -> Self { + self.credential.refresh = Some(refresh); + self + } +} + +#[cfg(test)] +mod tests { + use openshell_providers::parse_profile_yaml; + + use super::*; + use crate::policy::{Access, Endpoint as _}; + + fn parsed(yaml: &str) -> ProviderTypeProfile { + parse_profile_yaml(yaml).expect("the YAML parses") + } + + /// Each setter sets the YAML field it is named after, so the gateway + /// can't tell a built profile from the YAML. + #[test] + fn setters_match_the_yaml_they_name() { + let profile = Profile::new("example") + .display_name("Example") + .category(Category::Inference) + .credential( + Credential::new("api_key") + .env_var("API_KEY") + .env_var("OTHER_KEY") + .required() + .auth_style("header") + .header_name("x-api-key"), + ) + .discover(["api_key"]) + .endpoint("api.example.test", 443, |ep| { + ep.rest(Access::ReadWrite).enforce().path("/v1/**"); + }) + .endpoint("ws.example.test", 8443, |ep| { + ep.websocket(Access::ReadOnly); + }) + .binaries(["/usr/bin/curl", "/usr/bin/python*"]); + + let expected = parsed( + "id: example +display_name: Example +category: inference +credentials: + - name: api_key + env_vars: [API_KEY, OTHER_KEY] + required: true + auth_style: header + header_name: x-api-key +discovery: + credentials: [api_key] +endpoints: + - host: api.example.test + port: 443 + path: /v1/** + protocol: rest + access: read-write + enforcement: enforce + - host: ws.example.test + port: 8443 + protocol: websocket + access: read-only +binaries: [/usr/bin/curl, /usr/bin/python*] +", + ); + let yaml = profile.to_yaml().expect("renders"); + assert_eq!(parsed(&yaml), expected, "rendered:\n{yaml}"); + } + + /// A profile with only an id has the display name and category the + /// gateway requires. + #[test] + fn defaults() { + let yaml = Profile::new("bare").to_yaml().expect("renders"); + assert_eq!( + parsed(&yaml), + parsed("id: bare\ndisplay_name: bare\ncategory: other\n"), + "rendered:\n{yaml}" + ); + } + + #[test] + fn refresh() { + let refresh = proto::ProviderCredentialRefresh { + strategy: proto::ProviderCredentialRefreshStrategy::Oauth2RefreshToken as i32, + token_url: "https://issuer.example.test/token".to_string(), + scopes: vec!["openid".to_string()], + refresh_before: Some(prost_types::Duration { + seconds: 60, + nanos: 0, + }), + material: vec![proto::ProviderCredentialRefreshMaterial { + name: "refresh_token".to_string(), + required: true, + secret: true, + ..proto::ProviderCredentialRefreshMaterial::default() + }], + ..proto::ProviderCredentialRefresh::default() + }; + let yaml = Profile::new("refreshing") + .credential(Credential::new("token").refresh(refresh)) + .to_yaml() + .expect("renders"); + assert_eq!( + parsed(&yaml), + parsed( + "id: refreshing +display_name: refreshing +category: other +credentials: + - name: token + refresh: + strategy: oauth2_refresh_token + token_url: https://issuer.example.test/token + scopes: [openid] + refresh_before: 60s + material: + - name: refresh_token + required: true + secret: true +" + ), + "rendered:\n{yaml}" + ); + } +} From c7d83f854d49479409f5d947d4bf5c666c7d4224 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:21:23 -0800 Subject: [PATCH 08/63] test(e2e): build provider profiles with testkit::Profile Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 42 +++--- tests/suites/features/Cargo.lock | 18 ++- .../keycloak-provider-refresh/Cargo.toml | 2 +- .../tests/provider_refresh.rs | 127 ++++++------------ 4 files changed, 79 insertions(+), 110 deletions(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index c5b0cdfec8..537bb41b78 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -4,7 +4,8 @@ //! Providers attached to a sandbox: files a provider profile serves, and //! providers that `sandbox create --provider` creates from local credentials. -use testkit::{Context, ResultExt as _, YamlFile, policy}; +use testkit::policy::{self, Access, Endpoint as _}; +use testkit::{Context, Credential, Profile, ResultExt as _, YamlFile}; /// The example profile that serves `client.toml` from the provider's config. const FILES_PROFILE: &str = @@ -96,7 +97,7 @@ async fn auto_created_provider_credential_available_in_sandbox( cx: &mut Context, ) -> testkit::Result { let provider = cx - .import_profile(&auto_create_profile(&cx.name("auto"))) + .import_profile(&auto_create_profile(&cx.name("auto")).to_yaml()?) .await?; // The CLI names the provider it creates after the profile. let cli = cx.cli_handle(); @@ -176,28 +177,21 @@ async fn expect_project(sandbox: &testkit::Sandbox, path: &str, project: &str) - /// A profile `id` whose credential the CLI discovers in [`AUTO_CREATE_ENV`]. /// Its endpoint makes the gateway inject the credential without a policy /// binding; nothing connects to it. -fn auto_create_profile(id: &str) -> String { - format!( - "id: {id} -display_name: E2E Auto-created Provider -category: other -credentials: - - name: api_key - env_vars: [{AUTO_CREATE_ENV}] - required: true - auth_style: header - header_name: x-api-key -discovery: - credentials: [api_key] -endpoints: - - host: api.auto-provider.example - port: 443 - protocol: rest - access: read-write - enforcement: enforce -binaries: [/usr/bin/curl] -" - ) +fn auto_create_profile(id: &str) -> Profile { + Profile::new(id) + .display_name("E2E Auto-created Provider") + .credential( + Credential::new("api_key") + .env_var(AUTO_CREATE_ENV) + .required() + .auth_style("header") + .header_name("x-api-key"), + ) + .discover(["api_key"]) + .endpoint("api.auto-provider.example", 443, |ep| { + ep.rest(Access::ReadWrite).enforce(); + }) + .binaries(["/usr/bin/curl"]) } /// Whether `value` is the placeholder for the credential in `key`, in the diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index a92741823a..1df4ab86d2 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1833,7 +1833,7 @@ version = "0.0.0" dependencies = [ "keycloak-support", "openshell-testkit", - "tempfile", + "prost-types", ] [[package]] @@ -2428,6 +2428,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2479,6 +2494,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/suites/features/keycloak-provider-refresh/Cargo.toml b/tests/suites/features/keycloak-provider-refresh/Cargo.toml index e326fef5cb..1848d3cf86 100644 --- a/tests/suites/features/keycloak-provider-refresh/Cargo.toml +++ b/tests/suites/features/keycloak-provider-refresh/Cargo.toml @@ -14,4 +14,4 @@ publish = false [dependencies] keycloak-support = { path = "../../../support/keycloak" } openshell-testkit = { path = "../../../testkit/testkit" } -tempfile = "3" +prost-types = "0.14" diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index a617c24843..3c94f284d3 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -6,10 +6,10 @@ //! Keycloak is only the provider's token issuer here; the CLI still reaches //! the gateway the usual way. -use std::io::Write as _; - use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, ResultExt as _}; +use testkit::policy::{Access, Endpoint as _}; +use testkit::sdk::raw::proto; +use testkit::{CliOutput, Context, Credential, Profile}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -20,34 +20,14 @@ const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) -> testkit::Result { let kc = KeycloakContext::new(cx).await?; let provider = kc.name("kc-refresh"); - let profile_id = kc.name("kc-refresh-profile"); let grant = kc.keycloak().password_grant().await?; - let profile = write_profile(&profile_id, kc.keycloak())?; - let profile_path = profile.path().to_string_lossy().into_owned(); - - // Register each delete before its create, so cleanup also covers a create - // the gateway applied but the CLI reported as failed. Both deletes succeed - // when there is nothing to delete. - delete_on_cleanup(&kc, "profile", &profile_id); - kc.cli(["profile", "import", "--file", &profile_path]) - .await? - .success()?; - delete_on_cleanup(&kc, "provider", &provider); - kc.cli_with_env( + let profile = refresh_profile(&kc.name("kc-refresh-profile"), kc.keycloak()).to_yaml()?; + kc.install_provider( + &provider, + &profile, &[(CREDENTIAL_KEY, &grant.access_token)], - [ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--credential", - CREDENTIAL_KEY, - ], ) - .await? - .success()?; + .await?; let client_id = format!("client_id={}", kc.keycloak().client_id()); kc.cli_with_env( &[("KEYCLOAK_REFRESH_TOKEN", &grant.refresh_token)], @@ -117,64 +97,43 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - Ok(()) } -/// Delete the `kind` named `name` when the test ends, since testkit only -/// cleans up sandboxes by itself. -fn delete_on_cleanup(cx: &Context, kind: &'static str, name: &str) { - let cli = cx.cli_handle(); - let name = name.to_string(); - cx.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &name]).await?.success()?; - Ok(()) - }); -} - fn combined(output: &CliOutput) -> String { format!("{}{}", output.stdout, output.stderr) } -fn write_profile( - profile_id: &str, - keycloak: &keycloak_support::Keycloak, -) -> testkit::Result { - let issuer_token_url = keycloak.token_endpoint(); - let profile = format!( - r#"id: {profile_id} -display_name: Keycloak provider refresh test -category: other -credentials: - - name: access_token - env_vars: [{CREDENTIAL_KEY}] - required: true - auth_style: bearer - header_name: authorization - refresh: - strategy: oauth2_refresh_token - token_url: {issuer_token_url} - scopes: [openid] - refresh_before_seconds: 60 - max_lifetime_seconds: 3600 - material: - - name: client_id - required: true - - name: refresh_token - required: true - secret: true -endpoints: - - host: keycloak.test.invalid - port: 443 - protocol: rest - access: read-only - enforcement: enforce -binaries: - - /usr/bin/curl -"# - ); - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .context("create provider profile")?; - file.write_all(profile.as_bytes()) - .and_then(|()| file.flush()) - .context("write provider profile")?; - Ok(file) +/// A profile whose `access_token` the gateway refreshes from `keycloak` +/// with the OAuth refresh-token grant. +fn refresh_profile(id: &str, keycloak: &keycloak_support::Keycloak) -> Profile { + let material = |name: &str, secret: bool| proto::ProviderCredentialRefreshMaterial { + name: name.to_string(), + required: true, + secret, + ..proto::ProviderCredentialRefreshMaterial::default() + }; + let seconds = |seconds| Some(prost_types::Duration { seconds, nanos: 0 }); + Profile::new(id) + .display_name("Keycloak provider refresh test") + .credential( + Credential::new("access_token") + .env_var(CREDENTIAL_KEY) + .required() + .auth_style("bearer") + .header_name("authorization") + .refresh(proto::ProviderCredentialRefresh { + strategy: proto::ProviderCredentialRefreshStrategy::Oauth2RefreshToken as i32, + token_url: keycloak.token_endpoint(), + scopes: vec!["openid".to_string()], + refresh_before: seconds(60), + max_lifetime: seconds(3600), + material: vec![ + material("client_id", false), + material("refresh_token", true), + ], + ..proto::ProviderCredentialRefresh::default() + }), + ) + .endpoint("keycloak.test.invalid", 443, |ep| { + ep.rest(Access::ReadOnly).enforce(); + }) + .binaries(["/usr/bin/curl"]) } From b5ef8663d1a01d28c44054582871d64eb155973f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 10:31:33 -0800 Subject: [PATCH 09/63] test(conformance): run the auto-create sandbox as sleep infinity Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index 537bb41b78..dc0f689179 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -131,9 +131,8 @@ async fn auto_created_provider_credential_available_in_sandbox( &provider, "--auto-providers", "--", - "sh", - "-c", - "exec sleep infinity", + "sleep", + "infinity", ], ) .await? From d79cc8ad7ecc8c029ddbab8472515e93fa934f98 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 16:58:26 -0800 Subject: [PATCH 10/63] test(conformance): attach the provider files provider with SandboxBuilder::providers Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index dc0f689179..36fd784be9 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -28,7 +28,7 @@ async fn provider_file_open_update_and_detach(cx: &mut Context) -> testkit::Resu let sandbox = cx .sandbox("pf") - .spec(|spec| spec.providers = vec![provider.clone()]) + .providers([&provider]) .command(["sleep", "infinity"]) .create_ready() .await?; From edbaf953d8692c2f9e6b8419a4b0671d5e870cb0 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:37:43 -0800 Subject: [PATCH 11/63] test(testkit): add compute_driver, driver_config, sandbox logs, and policy set helpers Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 1 + .../conformance/src/credential_gating.rs | 26 +- tests/suites/drivers/podman/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 1 + tests/support/Cargo.lock | 1 + tests/testkit/Cargo.lock | 1 + tests/testkit/README.md | 27 +- tests/testkit/testkit/Cargo.toml | 1 + tests/testkit/testkit/src/context.rs | 264 +++++++++++++----- tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/logs.rs | 175 ++++++++++++ tests/testkit/testkit/src/policy.rs | 43 ++- 12 files changed, 445 insertions(+), 98 deletions(-) create mode 100644 tests/testkit/testkit/src/logs.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 76506381c3..8c4d9d10d2 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2420,6 +2420,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 24bffdbff9..0b4c4e6f29 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -13,9 +13,7 @@ use std::time::Duration; use host_server_support::{HttpRecorder, Request, TEST_HOST, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; -use testkit::{ - Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, -}; +use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; @@ -207,12 +205,12 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t let sandbox = create_sandbox(cx, "live", unbound.clone(), &[&provider]).await?; wait_for_token(&sandbox, "TOKEN_ABSENT").await?; - set_policy(cx, &sandbox, &bound) + policy::set(&sandbox, &bound) .await .context("bind the provider")?; wait_for_token(&sandbox, "TOKEN_PLACEHOLDER").await?; - set_policy(cx, &sandbox, &unbound) + policy::set(&sandbox, &unbound) .await .context("unbind the provider")?; wait_for_token(&sandbox, "TOKEN_ABSENT").await @@ -393,21 +391,3 @@ async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { Ok(()) } -/// Replace `sandbox`'s policy with `policy` and wait until it is applied. -async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { - let file = YamlFile::new(&policy::to_yaml(policy)?)?; - cx.cli([ - "policy", - "set", - sandbox.name(), - "--policy", - file.path()?, - "--wait", - "--timeout", - "120", - ]) - .await? - .success() - .context("policy set")?; - Ok(()) -} diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 560b02aa72..dc6e4cf989 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2408,6 +2408,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index b16d3fb67d..99c0081dd6 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2492,6 +2492,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 0883511b5c..1abb2da1d5 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2426,6 +2426,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index bad3c82e89..ee60aceb34 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2399,6 +2399,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 1bfd99b9f0..e2faeb5a3a 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -66,19 +66,21 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.sandbox(suffix)` to build a sandbox with a run-unique name (`test--`), then `create()` it or `create_ready()` to also wait for Ready. Set its image - with `image(..)`, its policy with `policy(..)` and its providers with - `providers(..)`. A long-running sandbox is - `.command(["sleep", "infinity"])`; add `.image(testkit::python_image())` - when a test runs Python clients in it. The returned + with `image(..)`, its policy with `policy(..)`, its providers with + `providers(..)`, and the template's driver-keyed `driver_config(json)`. A + long-running sandbox is `.command(["sleep", "infinity"])`; add + `.image(testkit::python_image())` when a test runs Python clients in it. The + returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set a setting, read `sandbox get` JSON with `get_json()`, read its policy revisions and active policy with `policy_revisions()` and - `active_policy()`, and delete. Its `expect_listed`, `expect_absent` and - `wait_absent` check list and get. `run_python(script, args)` runs a script - with `python3 -c` and fails unless it exits 0. Exec output has `success()`, - `failure()`, `expect_stdout()`, and `last_json()` for the last JSON object - or array line of stdout, such as a client script's result. + `active_policy()`, read its supervisor logs with `logs()` or wait for a + line with `logs_until(..)`, and delete. Its `expect_listed`, `expect_absent` + and `wait_absent` check list and get. `run_python(script, args)` runs a + script with `python3 -c` and fails unless it exits 0. Exec output has + `success()`, `failure()`, `expect_stdout()`, and `last_json()` for the last + JSON object or array line of stdout, such as a client script's result. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()` detaches and returns the `Sandbox`, `create_ready()` also waits for Ready, @@ -91,7 +93,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. create, so a create that fails after the gateway accepted it is still cleaned up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created. - `cx.expect_compute_driver(name)` to fail unless the gateway runs only that - compute driver. Driver suites call it first. + compute driver. Driver suites call it first. `cx.compute_driver()` returns + the driver's name, for setup that depends on it, such as a driver config. - `testkit::base_image()` for the image to create sandboxes from or build on: `OPENSHELL_TEST_BASE_IMAGE`, or OpenShell's default sandbox image. - `testkit::python_image()` for sandboxes that run `python3`, curl, git, or @@ -122,7 +125,9 @@ fails. A killed test process skips cleanup, so names are unique to the run. configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` for the CLI. `policy::binaries_yaml(binaries)` writes a `binaries:` list - into policy YAML a test writes by hand. + into policy YAML a test writes by hand. `policy::from_yaml` reads a policy + file, and `policy::set(&sandbox, &policy)` replaces a running sandbox's + policy and waits until it is loaded. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 7b8ebd90ab..d216c7b824 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -24,6 +24,7 @@ clap = { version = "4", features = ["derive"] } futures = "0.3" libtest-mimic = "0.8" linkme = "0.3" +prost-types = "0.14" quick-junit = "0.8" rand = "0.9" serde = { version = "1", features = ["derive"] } diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 0a671483ae..d2be88374c 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -187,6 +187,7 @@ impl Context { SandboxBuilder { handles: self.handles.clone(), policy: None, + driver_config: None, spec: SandboxSpec { name: Some(self.sandbox_name(suffix)), ..SandboxSpec::default() @@ -278,6 +279,18 @@ impl Context { /// gateway info reports it. Driver suites check this first, so a suite /// pointed at another driver's gateway fails instead of testing it. pub async fn expect_compute_driver(&self, name: &str) -> Result { + let driver = self.compute_driver().await?; + crate::ensure!( + driver == name, + "the gateway runs compute driver {driver}; expected {name}" + ); + Ok(()) + } + + /// The gateway's compute driver, such as `docker` or `kubernetes`, for + /// tests whose setup depends on it. Fails unless gateway info reports + /// exactly one driver, under the same configured and driver name. + pub async fn compute_driver(&self) -> Result { let info = self .handles .grpc @@ -287,22 +300,7 @@ impl Context { .await .context("get gateway info")? .into_inner(); - let drivers: Vec<(&str, &str)> = info - .compute_drivers - .iter() - .map(|driver| { - let reported = driver - .capabilities - .as_ref() - .map_or("", |capabilities| capabilities.driver_name.as_str()); - (driver.name.as_str(), reported) - }) - .collect(); - crate::ensure!( - matches!(drivers.as_slice(), [(configured, reported)] if *configured == name && *reported == name), - "the gateway reports compute drivers (name, driver name) {drivers:?}; expected only {name}" - ); - Ok(()) + single_compute_driver(&info) } /// Run the cleanup hooks, including deleting the sandboxes the test @@ -313,6 +311,30 @@ impl Context { } } +/// The one compute driver `info` reports, when its configured name and the +/// name the driver reports agree. +fn single_compute_driver(info: &proto::GetGatewayInfoResponse) -> Result { + let drivers: Vec<(&str, &str)> = info + .compute_drivers + .iter() + .map(|driver| { + let reported = driver + .capabilities + .as_ref() + .map_or("", |capabilities| capabilities.driver_name.as_str()); + (driver.name.as_str(), reported) + }) + .collect(); + match drivers.as_slice() { + [(configured, reported)] if configured == reported && !configured.is_empty() => { + Ok((*configured).to_string()) + } + _ => Err(Error::msg(format!( + "the gateway reports compute drivers (name, driver name) {drivers:?}; expected exactly one" + ))), + } +} + /// Register the cleanup hook that deletes sandbox `name`, unless it is /// already tracked. fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) { @@ -340,17 +362,19 @@ fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) /// The generated gRPC client, for public API the SDK does not wrap yet. #[derive(Clone)] -struct Grpc { +pub(crate) struct Grpc { client: OpenShellClient, workspace: String, } impl Grpc { - async fn client(&self) -> Result { + pub(crate) async fn client(&self) -> Result { Ok(self.client.raw_grpc_fresh().await?) } - fn scope(&self) -> Option { + pub(crate) fn scope( + &self, + ) -> Option { Some(proto::workspace_selector(&self.workspace)) } } @@ -370,6 +394,10 @@ impl Handles { &self.cli } + pub(crate) fn grpc(&self) -> &Grpc { + &self.grpc + } + pub(crate) fn track(&self, name: &str) { track_sandbox(&self.cleanup, &self.client, name); } @@ -394,6 +422,7 @@ impl Handles { pub struct SandboxBuilder { handles: Handles, policy: Option, + driver_config: Option, spec: SandboxSpec, } @@ -437,6 +466,20 @@ impl SandboxBuilder { self } + /// Set the template's `driver_config`, a JSON object keyed by driver + /// name, as `sandbox create --driver-config-json` does. The gateway + /// passes the block for its driver to the driver. + /// + /// ```ignore + /// let driver = cx.compute_driver().await?; + /// cx.sandbox("mount") + /// .driver_config(serde_json::json!({ driver: { "mounts": [] } })) + /// ``` + pub fn driver_config(mut self, config: serde_json::Value) -> Self { + self.driver_config = Some(config); + self + } + /// Full access to the spec for fields without a builder method. pub fn spec(mut self, update: impl FnOnce(&mut SandboxSpec)) -> Self { update(&mut self.spec); @@ -456,36 +499,40 @@ impl SandboxBuilder { pub async fn create(self) -> Result { let name = self.spec.name.clone().unwrap_or_default(); self.handles.track(&name); - let id = match self.policy { - None => { - self.handles - .client - .create_sandbox(self.spec) - .await - .context(format!("create sandbox {name}"))? - .id - } - // The SDK's spec has no policy field, so create through gRPC. - Some(policy) => { - crate::ensure!( - !self.spec.gpu && self.spec.service_exposures.is_empty(), - "testkit doesn't create a sandbox with a policy and a GPU or service exposures" - ); - let request = policy_create_request(self.spec, policy, self.handles.grpc.scope()); - self.handles - .grpc - .client() - .await? - .create_sandbox(request) - .await - .context(format!("create sandbox {name}"))?; - self.handles - .client - .get_sandbox(&name) - .await - .context(format!("get sandbox {name}"))? - .id - } + let id = if self.policy.is_none() && self.driver_config.is_none() { + self.handles + .client + .create_sandbox(self.spec) + .await + .context(format!("create sandbox {name}"))? + .id + } else { + // The SDK's spec has no policy or driver config, so create + // through gRPC. + crate::ensure!( + !self.spec.gpu && self.spec.service_exposures.is_empty(), + "testkit doesn't create a sandbox with a policy or driver config and a GPU or service exposures" + ); + let driver_config = self.driver_config.map(driver_config_struct).transpose()?; + let request = grpc_create_request( + self.spec, + self.policy, + driver_config, + self.handles.grpc.scope(), + ); + self.handles + .grpc + .client() + .await? + .create_sandbox(request) + .await + .context(format!("create sandbox {name}"))?; + self.handles + .client + .get_sandbox(&name) + .await + .context(format!("get sandbox {name}"))? + .id }; Ok(Sandbox { handles: self.handles, @@ -495,15 +542,28 @@ impl SandboxBuilder { } } -/// The gRPC request for a sandbox created from `spec` with `policy`, built -/// as the SDK builds one from `spec` alone. The spec is destructured without -/// `..`, so a field added to the SDK's spec fails to compile here instead of -/// being dropped. [`SandboxBuilder::create`] rejects a GPU or service -/// exposures before calling this. A policy field on the SDK's spec would -/// replace this (#5410). -fn policy_create_request( +/// `config` as the protobuf `Struct` the template carries. It must be a JSON +/// object, keyed by driver name. +fn driver_config_struct(config: serde_json::Value) -> Result { + let serde_json::Value::Object(fields) = config else { + return Err(Error::msg(format!( + "driver config must be a JSON object keyed by driver name; got {config}" + ))); + }; + openshell_core::proto_struct::json_object_to_struct(fields) + .map_err(|error| Error::msg(format!("driver config can't be represented: {error}"))) +} + +/// The gRPC request for a sandbox created from `spec` with `policy` and +/// `driver_config`, built as the SDK builds one from `spec` alone. The spec +/// is destructured without `..`, so a field added to the SDK's spec fails to +/// compile here instead of being dropped. [`SandboxBuilder::create`] rejects +/// a GPU or service exposures before calling this. Policy and driver config +/// fields on the SDK's spec would replace this (#5410). +fn grpc_create_request( spec: SandboxSpec, - policy: proto::SandboxPolicy, + policy: Option, + driver_config: Option, workspace_scope: Option, ) -> proto::CreateSandboxRequest { let SandboxSpec { @@ -524,14 +584,17 @@ fn policy_create_request( workspace_scope, spec: Some(proto::SandboxSpec { environment, - template: image.map(|image| proto::SandboxTemplate { - image, - ..proto::SandboxTemplate::default() + template: (image.is_some() || driver_config.is_some()).then(|| { + proto::SandboxTemplate { + image: image.unwrap_or_default(), + driver_config, + ..proto::SandboxTemplate::default() + } }), providers, command, tty, - policy: Some(policy), + policy, restart_policy: proto::SandboxRestartPolicy::from(restart_policy) as i32, ..proto::SandboxSpec::default() }), @@ -624,6 +687,10 @@ impl Sandbox { &self.name } + pub(crate) fn handles(&self) -> &Handles { + &self.handles + } + pub fn id(&self) -> &str { &self.id } @@ -1008,12 +1075,83 @@ mod tests { }; let policy = crate::policy::base(); - let spec = policy_create_request(spec, policy.clone(), None) + let spec = grpc_create_request(spec, Some(policy.clone()), None, None) .spec .expect("spec"); - assert_eq!(spec.template.expect("template").image, "python:3"); + let template = spec.template.expect("template"); + assert_eq!(template.image, "python:3"); + assert_eq!(template.driver_config, None); assert_eq!(spec.providers, ["provider"]); assert_eq!(spec.policy, Some(policy)); } + + #[test] + fn grpc_create_request_carries_driver_config_without_an_image() { + let config = driver_config_struct(serde_json::json!({ + "docker": { "mounts": [{ "type": "volume", "source": "data" }] } + })) + .expect("an object converts"); + + let spec = grpc_create_request(SandboxSpec::default(), None, Some(config.clone()), None) + .spec + .expect("spec"); + + let template = spec.template.expect("a template for the driver config"); + assert_eq!(template.image, "", "an empty image selects the default"); + assert_eq!(template.driver_config, Some(config)); + assert_eq!(spec.policy, None); + } + + #[test] + fn grpc_create_request_omits_an_empty_template() { + let spec = grpc_create_request(SandboxSpec::default(), None, None, None) + .spec + .expect("spec"); + assert_eq!(spec.template, None); + } + + #[test] + fn driver_config_must_be_an_object() { + let error = driver_config_struct(serde_json::json!(["docker"])).expect_err("an array"); + assert!( + error.to_string().contains("keyed by driver name"), + "{error}" + ); + } + + fn gateway_info(drivers: &[(&str, &str)]) -> proto::GetGatewayInfoResponse { + proto::GetGatewayInfoResponse { + compute_drivers: drivers + .iter() + .map(|(name, reported)| proto::ComputeDriverInfo { + name: (*name).to_string(), + capabilities: Some(proto::ComputeDriverCapabilities { + driver_name: (*reported).to_string(), + ..proto::ComputeDriverCapabilities::default() + }), + }) + .collect(), + ..proto::GetGatewayInfoResponse::default() + } + } + + #[test] + fn compute_driver_is_the_single_agreeing_driver() { + assert_eq!( + single_compute_driver(&gateway_info(&[("docker", "docker")])).expect("one driver"), + "docker" + ); + for drivers in [ + &[][..], + &[("docker", "podman")][..], + &[("", "")][..], + &[("docker", "docker"), ("podman", "podman")][..], + ] { + assert!( + single_compute_driver(&gateway_info(drivers)).is_err(), + "{drivers:?} should fail" + ); + } + } } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index e86b7c3b14..e24a9e6c60 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -34,6 +34,7 @@ mod context; mod error; pub mod gateway; mod image; +mod logs; mod output; pub mod policy; mod poll; @@ -55,6 +56,7 @@ pub use context::{ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; +pub use logs::{LOGS_TIMEOUT, Logs}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; diff --git a/tests/testkit/testkit/src/logs.rs b/tests/testkit/testkit/src/logs.rs new file mode 100644 index 0000000000..ce486fcc5b --- /dev/null +++ b/tests/testkit/testkit/src/logs.rs @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A sandbox's supervisor logs, as `openshell logs --source sandbox` shows +//! them. + +use std::fmt; +use std::time::Duration; + +use openshell_sdk::raw::proto; + +use crate::context::Sandbox; +use crate::error::{Result, ResultExt as _}; +use crate::poll::{Poll, poll_until}; + +/// The most lines [`Sandbox::logs`] returns, as `openshell logs -n 500`. +const LINES: u32 = 500; +/// How long [`Sandbox::logs_until`] waits. The supervisor pushes its logs to +/// the gateway in batches, so a line can appear a few seconds after the event. +pub const LOGS_TIMEOUT: Duration = Duration::from_secs(10); + +/// Log lines from a sandbox's supervisor, oldest first. +#[derive(Debug, Clone, Default)] +pub struct Logs { + lines: Vec, +} + +impl Logs { + pub fn new(lines: Vec) -> Self { + Self { lines } + } + + pub fn lines(&self) -> &[proto::SandboxLogLine] { + &self.lines + } + + /// Whether a line contains `needle` in its level, target, message, or a + /// field name or value, as `openshell logs` prints the line. + pub fn contains(&self, needle: &str) -> bool { + self.lines + .iter() + .any(|line| Line(line).to_string().contains(needle)) + } +} + +/// One line per log line, as `openshell logs` prints them. +impl fmt::Display for Logs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for line in &self.lines { + writeln!(f, "{}", Line(line))?; + } + Ok(()) + } +} + +/// `[.] [] [] [] `, +/// with fields sorted by name. +struct Line<'a>(&'a proto::SandboxLogLine); + +impl fmt::Display for Line<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let line = self.0; + let source = if line.source.is_empty() { + "gateway" + } else { + &line.source + }; + let (secs, millis) = line.event_time.as_ref().map_or((0, 0), |time| { + (time.seconds, time.nanos.clamp(0, 999_999_999) / 1_000_000) + }); + write!( + f, + "[{secs}.{millis:03}] [{source:<7}] [{:<5}] [{}] {}", + line.level, line.target, line.message + )?; + let mut fields: Vec<_> = line.fields.iter().collect(); + fields.sort_by_key(|(key, _)| key.as_str()); + for (key, value) in fields { + write!(f, " {key}={value}")?; + } + Ok(()) + } +} + +impl Sandbox { + /// The last 500 lines the sandbox's supervisor logged, as `openshell logs + /// -n 500 --source sandbox` returns them. + pub async fn logs(&self) -> Result { + let grpc = self.handles().grpc(); + let request = proto::GetSandboxLogsRequest { + sandbox: self.name().to_string(), + workspace_scope: grpc.scope(), + lines: LINES, + sources: vec!["sandbox".to_string()], + ..proto::GetSandboxLogsRequest::default() + }; + let response = grpc + .client() + .await? + .get_sandbox_logs(request) + .await + .context(format!("get logs of sandbox {}", self.name()))? + .into_inner(); + Ok(Logs::new(response.logs)) + } + + /// Read [`Sandbox::logs`] every 250 ms until `ready` accepts them, for up + /// to [`LOGS_TIMEOUT`]. A timeout reports the last logs read. + pub async fn logs_until(&self, what: &str, ready: impl Fn(&Logs) -> bool) -> Result { + poll_until( + &format!("sandbox {} to log {what}", self.name()), + LOGS_TIMEOUT, + Duration::from_millis(250), + async || { + let logs = self.logs().await?; + Ok(if ready(&logs) { + Poll::Ready(logs) + } else { + Poll::Pending(format!("logs:\n{logs}")) + }) + }, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn line(message: &str, source: &str, fields: &[(&str, &str)]) -> proto::SandboxLogLine { + proto::SandboxLogLine { + event_time: Some(prost_types::Timestamp { + seconds: 1234, + nanos: 567_000_000, + }), + level: "INFO".to_string(), + target: "openshell_supervisor".to_string(), + message: message.to_string(), + source: source.to_string(), + fields: fields + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ..proto::SandboxLogLine::default() + } + } + + #[test] + fn renders_lines_like_the_cli() { + let logs = Logs::new(vec![ + line("hello world", "sandbox", &[]), + line("denied", "", &[("dst_port", "443"), ("action", "deny")]), + ]); + assert_eq!( + logs.to_string(), + "[1234.567] [sandbox] [INFO ] [openshell_supervisor] hello world\n\ + [1234.567] [gateway] [INFO ] [openshell_supervisor] denied action=deny dst_port=443\n" + ); + } + + #[test] + fn contains_searches_message_target_and_fields_per_line() { + let logs = Logs::new(vec![ + line("first", "sandbox", &[("event", "endpoint_mismatch")]), + line("second", "sandbox", &[]), + ]); + assert!(logs.contains("first")); + assert!(logs.contains("openshell_supervisor")); + assert!(logs.contains("event=endpoint_mismatch")); + assert!(logs.contains("endpoint_mismatch")); + assert!(!logs.contains("first\n"), "a needle doesn't span lines"); + assert!(!logs.contains("absent")); + } +} diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index c62c930528..1f3a499779 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -3,7 +3,7 @@ //! Sandbox policies for tests: [`base`], read from `base-policy.yaml`, plus //! [`allow`] and the [`Endpoint`] settings for the network access a test -//! needs. +//! needs. [`set`] applies a policy to a running sandbox. //! //! ```ignore //! let mut policy = testkit::policy::base(); @@ -14,7 +14,9 @@ use openshell_sdk::raw::proto; +use crate::context::Sandbox; use crate::error::{Error, Result, ResultExt as _}; +use crate::yaml_file::YamlFile; pub use proto::NetworkAccessPreset as Access; @@ -87,6 +89,38 @@ pub fn binaries_yaml(binaries: &[&str]) -> String { format!("[{}]", paths.join(", ")) } +/// The policy `yaml` describes, as `sandbox create --policy` reads a file. +/// Use it for a policy kept as a YAML file, such as an example; build one +/// with [`base`] and [`allow`] otherwise. +pub fn from_yaml(yaml: &str) -> Result { + openshell_policy::parse_sandbox_policy(yaml) + .map_err(|error| Error::msg(format!("{error:?}"))) + .context("parse the policy") +} + +/// Replace `sandbox`'s policy with `policy` through `policy set --wait`, and +/// wait up to two minutes until the sandbox has loaded it. +pub async fn set(sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { + let file = YamlFile::new(&to_yaml(policy)?)?; + sandbox + .handles() + .cli() + .run([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .success() + .context(format!("set the policy of sandbox {}", sandbox.name()))?; + Ok(()) +} + /// Settings for an endpoint from [`allow`], named as in policy YAML. pub trait Endpoint { /// `protocol: rest` with `access`. @@ -196,5 +230,12 @@ mod tests { openshell_policy::parse_sandbox_policy(&to_yaml(&policy).expect("serializes")) .expect("the serialized policy parses"); assert_eq!(round_trip, expected); + assert_eq!(from_yaml(&yaml).expect("the YAML parses"), expected); + } + + #[test] + fn from_yaml_reports_an_invalid_policy() { + let error = from_yaml("network_policies: [").expect_err("invalid YAML"); + assert!(error.to_string().contains("parse the policy"), "{error}"); } } From af21f8dfc4a8c35f7c67792332059b7ef350d276 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:44:26 -0800 Subject: [PATCH 12/63] test(support): add handler responses, chunked bodies, and raw TCP servers to host-server Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 1 + tests/support/Cargo.lock | 1 + tests/support/host-server/Cargo.toml | 1 + tests/support/host-server/src/http.rs | 395 ++++++++++++++++++--- tests/support/host-server/src/lib.rs | 20 +- tests/support/host-server/src/tcp.rs | 174 +++++++++ tests/support/host-server/src/websocket.rs | 112 +++++- 8 files changed, 646 insertions(+), 59 deletions(-) create mode 100644 tests/support/host-server/src/tcp.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 8c4d9d10d2..9187a4cc8e 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1285,6 +1285,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 99c0081dd6..0e68a6149b 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1318,6 +1318,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 1abb2da1d5..044a44aa3e 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1282,6 +1282,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index e7db63f53d..cf51fd6a4c 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -13,6 +13,7 @@ publish = false [dependencies] openshell-testkit = { path = "../../testkit/testkit" } +serde_json = "1" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } tungstenite = "0.26" diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index 4d8ff8c812..5aec7932ff 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::fmt::Write as _; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; @@ -10,7 +12,7 @@ use tokio::task::JoinHandle; use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; -/// A request an [`HttpRecorder`] read. +/// A request an [`HttpRecorder`] read. A chunked body arrives decoded. #[derive(Clone, Debug)] pub struct Request { /// The request line and headers. @@ -18,26 +20,166 @@ pub struct Request { pub body: String, } -/// An HTTP server that answers every request with `200` and a fixed body, -/// and records each request it read. It reads a body by `Content-Length` -/// only, not a chunked one. +impl Request { + pub fn method(&self) -> &str { + self.request_line().0 + } + + /// The request target, such as `/path?query`. + pub fn path(&self) -> &str { + self.request_line().1 + } + + /// The value of the first header named `name`, in any case. + pub fn header(&self, name: &str) -> Option { + header(&self.head, name) + } + + fn request_line(&self) -> (&str, &str) { + let line = self.head.lines().next().unwrap_or_default(); + let mut parts = line.split(' '); + ( + parts.next().unwrap_or_default(), + parts.next().unwrap_or_default(), + ) + } +} + +/// A response an [`HttpRecorder`] handler returns. The server adds +/// `Content-Length`, except to an informational response, and `Connection`, +/// unless the handler set one. +#[derive(Clone, Debug)] +pub struct Response { + status: u16, + headers: Vec<(String, String)>, + body: Vec, + keep_alive: bool, +} + +impl Response { + /// A response with `status` and no body, that closes the connection. + pub fn new(status: u16) -> Self { + Self { + status, + headers: Vec::new(), + body: Vec::new(), + keep_alive: false, + } + } + + /// A `200` response with a `text/plain` body. + pub fn text(body: impl Into) -> Self { + Self::new(200) + .header("Content-Type", "text/plain") + .body(body.into()) + } + + /// A `200` response with an `application/json` body. + pub fn json(body: &serde_json::Value) -> Self { + Self::new(200) + .header("Content-Type", "application/json") + .body(body.to_string()) + } + + #[must_use] + pub fn status(mut self, status: u16) -> Self { + self.status = status; + self + } + + #[must_use] + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + #[must_use] + pub fn body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } + + /// Keep the connection open for another request, unless the request + /// asked to close it. + #[must_use] + pub fn keep_alive(mut self) -> Self { + self.keep_alive = true; + self + } + + fn encode(&self, close: bool) -> Vec { + let mut head = format!("HTTP/1.1 {} {}\r\n", self.status, reason(self.status)); + for (name, value) in &self.headers { + let _ = write!(head, "{name}: {value}\r\n"); + } + if self.status >= 200 { + let _ = write!(head, "Content-Length: {}\r\n", self.body.len()); + } + if !self + .headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("connection")) + { + let connection = if close { "close" } else { "keep-alive" }; + let _ = write!(head, "Connection: {connection}\r\n"); + } + head.push_str("\r\n"); + let mut bytes = head.into_bytes(); + bytes.extend_from_slice(&self.body); + bytes + } +} + +fn reason(status: u16) -> &'static str { + match status { + 101 => "Switching Protocols", + 200 => "OK", + 202 => "Accepted", + 204 => "No Content", + 400 => "Bad Request", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 418 => "I'm a teapot", + _ => "Status", + } +} + +type Handler = dyn Fn(&Request) -> Response + Send + Sync; + +/// An HTTP server that records each request it read and answers it with a +/// handler's response. A connection stays open for another request while +/// the responses keep it alive and the requests don't close it. pub struct HttpRecorder { pub port: u16, requests: Arc>>, + connections: Arc, task: JoinHandle<()>, } impl HttpRecorder { + /// Answer every request with `200` and `body`. pub async fn start(body: impl Into) -> Result { + let body = body.into(); + Self::serve(move |_| Response::text(body.clone())).await + } + + /// Answer every request with `handler`'s response. + pub async fn serve( + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { let (listener, port) = bind().await?; - let body: Arc = body.into().into(); + let handler: Arc = Arc::new(handler); let requests = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); let task_requests = Arc::clone(&requests); + let task_connections = Arc::clone(&connections); let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); tokio::spawn(respond( stream, - Arc::clone(&body), + Arc::clone(&handler), Arc::clone(&task_requests), )); } @@ -45,10 +187,24 @@ impl HttpRecorder { Ok(Self { port, requests, + connections, task, }) } + /// The requests the server has read so far. + pub fn requests(&self) -> Vec { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The number of connections the server has accepted. + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Acquire) + } + /// The requests the server has read, once there are at least `count`. pub async fn wait_for_requests(&self, count: usize) -> Result> { testkit::poll_until( @@ -59,11 +215,7 @@ impl HttpRecorder { WAIT_TIMEOUT, WAIT_INTERVAL, async || { - let requests = self - .requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); + let requests = self.requests(); Ok(if requests.len() >= count { Poll::Ready(requests) } else { @@ -81,45 +233,204 @@ impl Drop for HttpRecorder { } } -/// Read one request, headers and `Content-Length` body, record it, and -/// answer with `body`. -async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { +/// Read requests, record each, and answer with `handler` until a response +/// or request closes the connection. +async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc>>) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { - return; - }; - requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(request); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - let _ = stream.write_all(response.as_bytes()).await; + let mut buffer = Vec::new(); + while let Some(request) = read_request(&mut stream, &mut buffer).await { + let response = handler(&request); + let close = !response.keep_alive + || request + .header("connection") + .is_some_and(|value| value.eq_ignore_ascii_case("close")); + requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(request); + if stream.write_all(&response.encode(close)).await.is_err() || close { + break; + } + } let _ = stream.shutdown().await; } -pub(crate) async fn read_request(stream: &mut TcpStream) -> Option { - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; +/// Read one request from `stream`, starting with the bytes already in +/// `buffer`, and leave any bytes after it in `buffer`. +pub(crate) async fn read_request(stream: &mut TcpStream, buffer: &mut Vec) -> Option { + let mut chunk = [0_u8; 4096]; loop { - if let Some(end) = find(&request, b"\r\n\r\n") { - let head = String::from_utf8_lossy(&request[..end]).into_owned(); - let length = header(&head, "content-length") - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - if request.len() >= end + 4 + length { - let body = - String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); - return Some(Request { head, body }); - } + if let Some((request, length)) = parse_request(buffer) { + buffer.drain(..length); + return Some(request); } - match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { - Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { - request.extend_from_slice(&buffer[..read]); + match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { + Ok(Ok(read)) if read > 0 && buffer.len() < MAX_REQUEST => { + buffer.extend_from_slice(&chunk[..read]); } _ => return None, } } } + +/// A complete request at the start of `bytes`, and its length. +fn parse_request(bytes: &[u8]) -> Option<(Request, usize)> { + let end = find(bytes, b"\r\n\r\n")?; + let head = String::from_utf8_lossy(&bytes[..end]).into_owned(); + let rest = &bytes[end + 4..]; + let chunked = header(&head, "transfer-encoding") + .is_some_and(|value| value.to_ascii_lowercase().contains("chunked")); + let (body, length) = if chunked { + decode_chunked(rest)? + } else { + let length = header(&head, "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + (rest.get(..length)?.to_vec(), length) + }; + let body = String::from_utf8_lossy(&body).into_owned(); + Some((Request { head, body }, end + 4 + length)) +} + +/// A complete chunked body at the start of `bytes`, decoded, and its encoded +/// length. +fn decode_chunked(bytes: &[u8]) -> Option<(Vec, usize)> { + let mut body = Vec::new(); + let mut position = 0; + loop { + let line_end = position + find(&bytes[position..], b"\r\n")?; + let size = std::str::from_utf8(&bytes[position..line_end]).ok()?; + let size = size.split(';').next().unwrap_or_default().trim(); + let size = usize::from_str_radix(size, 16).ok()?; + position = line_end + 2; + if size == 0 { + // Skip trailers up to the blank line that ends the body. + loop { + let line_end = position + find(&bytes[position..], b"\r\n")?; + let blank = line_end == position; + position = line_end + 2; + if blank { + return Some((body, position)); + } + } + } + body.extend_from_slice(bytes.get(position..position + size)?); + position += size; + if bytes.get(position..position + 2)? != b"\r\n" { + return None; + } + position += 2; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect(port: u16) -> TcpStream { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + stream.set_nodelay(true).unwrap(); + stream + } + + async fn read_to_end(stream: &mut TcpStream) -> String { + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + String::from_utf8_lossy(&response).into_owned() + } + + #[test] + fn parse_request_decodes_a_chunked_body_and_leaves_the_next_request() { + let bytes = b"POST /a HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n\ + 3;ext=1\r\nabc\r\nA\r\n0123456789\r\n0\r\nTrailer: x\r\n\r\nGET /b"; + let (request, length) = parse_request(bytes).unwrap(); + assert_eq!(request.method(), "POST"); + assert_eq!(request.path(), "/a"); + assert_eq!(request.body, "abc0123456789"); + assert_eq!(&bytes[length..], b"GET /b"); + } + + #[test] + fn parse_request_waits_for_the_whole_body() { + assert!(parse_request(b"POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nabc").is_none()); + assert!( + parse_request(b"POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n") + .is_none() + ); + let (request, _) = + parse_request(b"POST / HTTP/1.1\r\ncontent-length: 4\r\n\r\nabcd").unwrap(); + assert_eq!(request.body, "abcd"); + assert_eq!(request.header("Content-Length").as_deref(), Some("4")); + } + + #[test] + fn response_encodes_length_and_connection_unless_set() { + let encoded = String::from_utf8(Response::text("ok").encode(true)).unwrap(); + assert_eq!( + encoded, + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok" + ); + let upgrade = Response::new(101) + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .encode(true); + assert_eq!( + String::from_utf8(upgrade).unwrap(), + "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n" + ); + } + + #[tokio::test] + async fn http_recorder_keeps_a_connection_alive_until_the_request_closes_it() { + let recorder = + HttpRecorder::serve(|request| Response::text(request.path().to_string()).keep_alive()) + .await + .unwrap(); + let mut stream = connect(recorder.port).await; + stream + .write_all( + b"GET /one HTTP/1.1\r\nHost: test\r\n\r\n\ + GET /two HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let response = read_to_end(&mut stream).await; + assert!( + response.contains("Connection: keep-alive\r\n\r\n/one"), + "{response}" + ); + assert!( + response.ends_with("Connection: close\r\n\r\n/two"), + "{response}" + ); + let paths: Vec<_> = recorder + .wait_for_requests(2) + .await + .unwrap() + .iter() + .map(|request| request.path().to_string()) + .collect(); + assert_eq!(paths, ["/one", "/two"]); + assert_eq!(recorder.connections(), 1); + } + + #[tokio::test] + async fn http_recorder_closes_after_a_response_that_is_not_kept_alive() { + let recorder = HttpRecorder::start("body").await.unwrap(); + let mut stream = connect(recorder.port).await; + stream + .write_all(b"GET / HTTP/1.1\r\nHost: test\r\n\r\nGET /ignored HTTP/1.1\r\n\r\n") + .await + .unwrap(); + let response = read_to_end(&mut stream).await; + assert!( + response.ends_with("Connection: close\r\n\r\nbody"), + "{response}" + ); + assert_eq!(recorder.requests().len(), 1); + } +} diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index f10a0e0632..5df228f90a 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -9,8 +9,17 @@ //! `host.openshell.internal` to different ones, and stop when dropped. They //! run where the tests run, so a gateway whose sandboxes can't reach the test //! host, such as one on a remote cluster, can't run tests that use them. +//! +//! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's +//! [`Response`], and records each parsed [`Request`]. +//! - [`WebSocketRecorder`] completes the WebSocket handshake and records the +//! frames that arrive. +//! - [`TcpServer`] records raw bytes, and echoes them or answers once the +//! client goes idle. +//! - [`free_port`] finds a port nothing listens on. mod http; +mod tcp; mod websocket; use std::time::Duration; @@ -18,8 +27,9 @@ use std::time::Duration; use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; -pub use http::{HttpRecorder, Request}; -pub use websocket::WebSocketRecorder; +pub use http::{HttpRecorder, Request, Response}; +pub use tcp::TcpServer; +pub use websocket::{TEXT_REPLY, WebSocketRecorder}; /// The name a sandbox reaches these servers by. pub const TEST_HOST: &str = "host.openshell.internal"; @@ -40,6 +50,12 @@ async fn bind() -> Result<(TcpListener, u16)> { Ok((listener, port)) } +/// A port that nothing on the test host listened on when this returned. +pub async fn free_port() -> Result { + let (_listener, port) = bind().await?; + Ok(port) +} + fn find(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs new file mode 100644 index 0000000000..42c4eaa9b0 --- /dev/null +++ b/tests/support/host-server/src/tcp.rs @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find}; + +/// How long a [`TcpServer::reply_when_idle`] connection waits for more bytes +/// before it answers. +const IDLE: Duration = Duration::from_millis(200); + +/// A TCP server that records the bytes every connection sent, in arrival +/// order, for a test that checks raw bytes rather than parsed requests. +pub struct TcpServer { + pub port: u16, + received: Arc>>, + connections: Arc, + task: JoinHandle<()>, +} + +#[derive(Clone)] +enum Mode { + Echo, + ReplyWhenIdle(Arc<[u8]>), +} + +impl TcpServer { + /// Write back every byte a connection sends until it closes. + pub async fn echo() -> Result { + Self::start(Mode::Echo).await + } + + /// Read until a connection is idle for 200 ms or closes, then write + /// `reply` and close it. A test can send several requests at once and + /// see all of them, since the server answers none before the last. + pub async fn reply_when_idle(reply: impl Into>) -> Result { + Self::start(Mode::ReplyWhenIdle(reply.into().into())).await + } + + async fn start(mode: Mode) -> Result { + let (listener, port) = bind().await?; + let received = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); + let task_received = Arc::clone(&received); + let task_connections = Arc::clone(&connections); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); + tokio::spawn(serve(stream, mode.clone(), Arc::clone(&task_received))); + } + }); + Ok(Self { + port, + received, + connections, + task, + }) + } + + /// The bytes the server has read so far. + pub fn received(&self) -> Vec { + self.received + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The number of connections the server has accepted. + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Acquire) + } + + /// The bytes the server has read, once they contain `needle`. + pub async fn wait_for(&self, needle: &[u8]) -> Result> { + testkit::poll_until( + &format!( + "{:?} at the host server on port {}", + String::from_utf8_lossy(needle), + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + let received = self.received(); + Ok(if find(&received, needle).is_some() { + Poll::Ready(received) + } else { + Poll::Pending(format!("{} byte(s)", received.len())) + }) + }, + ) + .await + } +} + +impl Drop for TcpServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn serve(mut stream: TcpStream, mode: Mode, received: Arc>>) { + let _ = stream.set_nodelay(true); + let idle = match mode { + Mode::Echo => READ_TIMEOUT, + Mode::ReplyWhenIdle(_) => IDLE, + }; + let mut buffer = [0_u8; 4096]; + let mut total = 0; + loop { + let read = match tokio::time::timeout(idle, stream.read(&mut buffer)).await { + Ok(Ok(read)) if read > 0 && total < MAX_REQUEST => read, + _ => break, + }; + total += read; + received + .lock() + .unwrap_or_else(PoisonError::into_inner) + .extend_from_slice(&buffer[..read]); + if matches!(mode, Mode::Echo) && stream.write_all(&buffer[..read]).await.is_err() { + return; + } + } + if let Mode::ReplyWhenIdle(reply) = mode { + let _ = stream.write_all(&reply).await; + } + let _ = stream.shutdown().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect(port: u16) -> TcpStream { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + stream.set_nodelay(true).unwrap(); + stream + } + + #[tokio::test] + async fn echo_writes_back_and_records_bytes() { + let server = TcpServer::echo().await.unwrap(); + let mut stream = connect(server.port).await; + stream.write_all(&[0x00, 0xff, b'x']).await.unwrap(); + let mut echoed = [0_u8; 3]; + stream.read_exact(&mut echoed).await.unwrap(); + assert_eq!(echoed, [0x00, 0xff, b'x']); + assert_eq!(server.wait_for(b"x").await.unwrap(), [0x00, 0xff, b'x']); + assert_eq!(server.connections(), 1); + } + + #[tokio::test] + async fn reply_when_idle_records_everything_before_replying() { + let server = TcpServer::reply_when_idle(b"reply".to_vec()).await.unwrap(); + let mut stream = connect(server.port).await; + stream.write_all(b"first ").await.unwrap(); + stream.write_all(b"second").await.unwrap(); + let mut reply = Vec::new(); + stream.read_to_end(&mut reply).await.unwrap(); + assert_eq!(reply, b"reply"); + assert_eq!(server.received(), b"first second"); + } +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 264dd4b5d3..4f38bdb8f3 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -10,21 +10,26 @@ use tokio::net::TcpStream; use tokio::task::JoinHandle; use crate::http::read_request; -use crate::{READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; /// A WebSocket server that completes the handshake, then reads frames until -/// the connection closes, and records whether any was a binary frame. +/// the connection closes. It records whether any was a binary frame, records +/// each text frame, and answers each text frame with [`TEXT_REPLY`]. pub struct WebSocketRecorder { pub port: u16, state: Arc, task: JoinHandle<()>, } +/// The text frame a [`WebSocketRecorder`] sends after each text frame. +pub const TEXT_REPLY: &str = "received"; + #[derive(Default)] struct WebSocketState { handshake_seen: AtomicBool, binary_seen: AtomicBool, finished: AtomicBool, + texts: Mutex>, } impl WebSocketRecorder { @@ -81,6 +86,15 @@ impl WebSocketRecorder { ) .await } + + /// The text frames the server has read so far. + pub fn texts(&self) -> Vec { + self.state + .texts + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } } impl Drop for WebSocketRecorder { @@ -91,7 +105,7 @@ impl Drop for WebSocketRecorder { async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { + let Some(request) = read_request(&mut stream, &mut Vec::new()).await else { return; }; let Some(key) = header(&request.head, "sec-websocket-key") else { @@ -111,8 +125,8 @@ async fn accept_websocket(mut stream: TcpStream, state: Arc) { state.finished.store(true, Ordering::Release); } -/// Read frames until a close frame or the end of the stream, and record a -/// binary frame. +/// Read frames until a close frame or the end of the stream, record a +/// binary frame and each text frame, and answer each text frame. async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { loop { let mut frame_header = [0_u8; 2]; @@ -123,31 +137,77 @@ async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io: 127 => stream.read_u64().await?, length => u64::from(length), }; - if opcode == 0x02 { - state.binary_seen.store(true, Ordering::Release); - } - if opcode == 0x08 { - return Ok(()); + match opcode { + 0x01 => { + let text = read_payload(stream, frame_header[1], length).await?; + state + .texts + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(String::from_utf8_lossy(&text).into_owned()); + stream.write_all(&server_text_frame(TEXT_REPLY)).await?; + continue; + } + 0x02 => state.binary_seen.store(true, Ordering::Release), + _ => {} } let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; + // Read a close frame's payload too, since closing with unread bytes + // resets the connection. let skipped = tokio::io::copy( &mut (&mut *stream).take(mask + length), &mut tokio::io::sink(), ) .await?; - if skipped < mask + length { + if opcode == 0x08 || skipped < mask + length { return Ok(()); } } } +/// Read a text frame's payload, unmasked, up to [`MAX_REQUEST`] bytes. +async fn read_payload( + stream: &mut TcpStream, + second_byte: u8, + length: u64, +) -> std::io::Result> { + let length = usize::try_from(length) + .ok() + .filter(|length| *length <= MAX_REQUEST) + .ok_or_else(|| std::io::Error::other("text frame too large"))?; + let mut mask = [0_u8; 4]; + if second_byte & 0x80 != 0 { + stream.read_exact(&mut mask).await?; + } + let mut payload = vec![0_u8; length]; + stream.read_exact(&mut payload).await?; + for (index, byte) in payload.iter_mut().enumerate() { + *byte ^= mask[index % 4]; + } + Ok(payload) +} + +/// A final, unmasked server text frame. +fn server_text_frame(text: &str) -> Vec { + let mut frame = vec![0x81]; + match text.len() { + length @ 0..=125 => frame.push(u8::try_from(length).unwrap_or_default()), + length => { + frame.push(126); + frame.extend_from_slice(&u16::try_from(length).unwrap_or(u16::MAX).to_be_bytes()); + } + } + frame.extend_from_slice(text.as_bytes()); + frame +} + #[cfg(test)] mod tests { use super::*; use crate::find; /// Complete a handshake with `recorder`, send `frames`, and close. - async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { + async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) -> TcpStream { #[allow( clippy::disallowed_methods, reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" @@ -181,6 +241,7 @@ mod tests { stream.write_all(&frame(*opcode, payload)).await.unwrap(); } stream.write_all(&frame(0x08, &[])).await.unwrap(); + stream } /// A final, masked client frame. @@ -202,22 +263,43 @@ mod tests { #[tokio::test] async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( + // Hold the connection so the replies to text frames don't reset it. + let _stream = send_frames( &recorder, &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], ) .await; assert!(recorder.wait_until_finished().await.unwrap()); + assert_eq!(recorder.texts(), ["x".repeat(300)]); } #[tokio::test] async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( + // Hold the connection so the replies to text frames don't reset it. + let _stream = send_frames( &recorder, &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], ) .await; assert!(!recorder.wait_until_finished().await.unwrap()); } + + #[tokio::test] + async fn websocket_recorder_answers_each_text_frame() { + let recorder = WebSocketRecorder::start().await.unwrap(); + let mut stream = send_frames( + &recorder, + &[(0x01, b"one".to_vec()), (0x01, b"two".to_vec())], + ) + .await; + let mut replies = Vec::new(); + stream.read_to_end(&mut replies).await.unwrap(); + assert_eq!( + replies, + [server_text_frame(TEXT_REPLY), server_text_frame(TEXT_REPLY)].concat() + ); + recorder.wait_until_finished().await.unwrap(); + assert_eq!(recorder.texts(), ["one", "two"]); + } } From 7ceb1b4ad9dd9887c80e310e58d56c8ae5fb104e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:45:45 -0800 Subject: [PATCH 13/63] test(testkit): add policy::base_with for YAML policy sections Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 7 ++++--- tests/testkit/testkit/src/policy.rs | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index e2faeb5a3a..50849cff01 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -125,9 +125,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` for the CLI. `policy::binaries_yaml(binaries)` writes a `binaries:` list - into policy YAML a test writes by hand. `policy::from_yaml` reads a policy - file, and `policy::set(&sandbox, &policy)` replaces a running sandbox's - policy and waits until it is loaded. + into policy YAML a test writes by hand. `policy::base_with(yaml)` adds YAML + sections, such as L7 rules, to the base policy. `policy::from_yaml` reads a + policy file, and `policy::set(&sandbox, &policy)` replaces a running + sandbox's policy and waits until it is loaded. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index 1f3a499779..69e6ce3bcf 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -98,6 +98,13 @@ pub fn from_yaml(yaml: &str) -> Result { .context("parse the policy") } +/// [`base`] with the top-level sections in `yaml`, such as +/// `network_policies` or `network_middlewares`, added. Use it for policy +/// settings [`Endpoint`] doesn't cover, such as L7 rules. +pub fn base_with(yaml: &str) -> Result { + from_yaml(&format!("{BASE}{yaml}")) +} + /// Replace `sandbox`'s policy with `policy` through `policy set --wait`, and /// wait up to two minutes until the sandbox has loaded it. pub async fn set(sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { @@ -231,6 +238,11 @@ mod tests { .expect("the serialized policy parses"); assert_eq!(round_trip, expected); assert_eq!(from_yaml(&yaml).expect("the YAML parses"), expected); + assert_eq!( + base_with(yaml.strip_prefix(BASE).expect("starts with the base")) + .expect("the YAML parses"), + expected + ); } #[test] From fe33f62f610307f0d28d3118cf9d669f1342c769 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:28 -0800 Subject: [PATCH 14/63] test(conformance): port network bypass, listener, and loopback tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 2 + tests/suites/conformance/src/network.rs | 133 ++++++++++++++++++ .../conformance/src/scripts/bypass_client.py | 26 ++++ .../src/scripts/destination_denials_client.py | 21 +++ .../src/scripts/listener_lifetime_probe.py | 81 +++++++++++ .../src/scripts/localhost_client.py | 38 +++++ .../src/scripts/preconnect_owner.py | 21 +++ .../src/scripts/socket_not_inherited.py | 14 ++ tests/suites/conformance/src/support.rs | 79 +++++++++++ 9 files changed, 415 insertions(+) create mode 100644 tests/suites/conformance/src/network.rs create mode 100644 tests/suites/conformance/src/scripts/bypass_client.py create mode 100644 tests/suites/conformance/src/scripts/destination_denials_client.py create mode 100644 tests/suites/conformance/src/scripts/listener_lifetime_probe.py create mode 100644 tests/suites/conformance/src/scripts/localhost_client.py create mode 100644 tests/suites/conformance/src/scripts/preconnect_owner.py create mode 100644 tests/suites/conformance/src/scripts/socket_not_inherited.py create mode 100644 tests/suites/conformance/src/support.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 8ac23f54bc..f01f71f9b5 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -18,11 +18,13 @@ mod filesystem_policy; mod image; mod labels; mod lifecycle; +mod network; mod persistent_workspace; mod policy_advisor; mod process; mod settings; mod smoke; mod ssh; +mod support; mod templates; mod workspaces; diff --git a/tests/suites/conformance/src/network.rs b/tests/suites/conformance/src/network.rs new file mode 100644 index 0000000000..b618509fd6 --- /dev/null +++ b/tests/suites/conformance/src/network.rs @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox networking: direct connections the sandbox refuses, loopback +//! traffic it leaves alone, and listening sockets whose lifetime it must not +//! extend. +//! +//! Ported from the `bypass_detection`, `listener_lifetime`, and `no_proxy` +//! e2e tests, and the destination denials in `proxy_egress_pipeline`. + +use std::time::Duration; + +use serde_json::json; +use testkit::{Context, ResultExt as _, policy}; + +use crate::support::{ + expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const BYPASS_CLIENT: &str = include_str!("scripts/bypass_client.py"); +const DESTINATION_DENIALS_CLIENT: &str = include_str!("scripts/destination_denials_client.py"); +const LOCALHOST_CLIENT: &str = include_str!("scripts/localhost_client.py"); +const LISTENER_LIFETIME_PROBE: &str = include_str!("scripts/listener_lifetime_probe.py"); +const PRECONNECT_OWNER: &str = include_str!("scripts/preconnect_owner.py"); +const SOCKET_NOT_INHERITED: &str = include_str!("scripts/socket_not_inherited.py"); + +/// A direct TCP connection that bypasses the proxy is denied with a +/// permission error well before the socket's 10-second timeout. +#[testkit::test] +async fn direct_connect_bypass_is_denied_promptly(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "bypass", policy::base()).await?; + let output = run_python(&sandbox, BYPASS_CLIENT, &[]).await?; + let result = json_result(&output)?; + testkit::ensure!( + result["bypass_result"] == "denied", + "expected the direct connect to be denied; got {result}" + ); + let elapsed = result["elapsed_ms"].as_u64().unwrap_or(u64::MAX); + testkit::ensure!( + elapsed < 8000, + "the denial took {elapsed} ms; expected under 8000" + ); + Ok(()) +} + +/// Connections to cloud metadata, a Kubernetes API port, and an address +/// outside an endpoint's `allowed_ips` fail with `EACCES` even though the +/// policy names them. +#[testkit::test] +async fn reserved_destinations_fail_with_eacces_even_when_allowed( + cx: &mut Context, +) -> testkit::Result { + let policy = policy::base_with(&format!( + r#"network_policies: + destination_denials: + name: destination_denials + endpoints: + - {{ host: 169.254.169.254, port: 80 }} + - {{ host: 127.0.0.1, port: 80 }} + - {{ host: 203.0.113.10, port: 6443 }} + - host: 203.0.113.10 + port: 8080 + allowed_ips: ["198.51.100.0/24"] + binaries: {binaries} +"#, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "denials", policy).await?; + let output = run_python(&sandbox, DESTINATION_DENIALS_CLIENT, &[]).await?; + expect_values( + &json_result(&output)?, + &[ + ("metadata", json!(13)), + ("control_plane", json!(13)), + ("outside_allowed_ips", json!(13)), + ], + ) +} + +/// A process can serve and fetch HTTP on 127.0.0.1 without proxy variables +/// in its environment. +#[testkit::test] +async fn localhost_is_reachable_without_proxy_environment(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "local", policy::base()).await?; + let output = run_python(&sandbox, LOCALHOST_CLIENT, &[]).await?; + expect_values( + &json_result(&output)?, + &[ + ("proxy_env_absent", json!(true)), + ("payload", json!({"message": "hello"})), + ], + ) +} + +/// Closing a listener, or exiting while holding one, frees its port: new +/// connections are refused and the port can be bound again. A later exec +/// doesn't inherit the exited child's listener. +#[testkit::test] +async fn closed_or_exited_listener_releases_its_port(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "listen", policy::base()).await?; + let output = run_python(&sandbox, LISTENER_LIFETIME_PROBE, &[]).await?; + let inode = json_result(&output)?[1].to_string(); + run_python(&sandbox, SOCKET_NOT_INHERITED, &[&inode]) + .await + .context("a later exec inherited the closed listener")?; + Ok(()) +} + +/// An exec doesn't inherit a socket that the sandbox's main process opened +/// without close-on-exec. +#[testkit::test] +async fn exec_does_not_inherit_main_process_socket(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("preconn") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["python3", "-c", PRECONNECT_OWNER]) + .create_ready() + .await?; + let inode = sandbox + .poll_exec( + "the main process to publish its socket inode", + Duration::from_secs(30), + ["cat", "/sandbox/preconnect-inode"], + |output| output.exit_code == 0 && output.stdout.trim().parse::().is_ok(), + ) + .await? + .stdout; + run_python(&sandbox, SOCKET_NOT_INHERITED, &[inode.trim()]) + .await + .context("the exec inherited the main process's socket")?; + Ok(()) +} diff --git a/tests/suites/conformance/src/scripts/bypass_client.py b/tests/suites/conformance/src/scripts/bypass_client.py new file mode 100644 index 0000000000..e3c1a75d51 --- /dev/null +++ b/tests/suites/conformance/src/scripts/bypass_client.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects straight to an unroutable address, bypassing any proxy, and +# prints the outcome and how long it took. +import json, socket, time + +start = time.monotonic() +result = "unknown" +try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(10) + s.connect(("198.51.100.1", 80)) + result = "connected" + s.close() +except ConnectionRefusedError: + result = "refused" +except PermissionError: + result = "denied" +except socket.timeout: + result = "timeout" +except OSError as e: + result = f"error:{e}" + +elapsed_ms = int((time.monotonic() - start) * 1000) +print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=True) diff --git a/tests/suites/conformance/src/scripts/destination_denials_client.py b/tests/suites/conformance/src/scripts/destination_denials_client.py new file mode 100644 index 0000000000..a4ddd6475e --- /dev/null +++ b/tests/suites/conformance/src/scripts/destination_denials_client.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects to addresses the policy names but the sandbox must still refuse, +# and prints each connect's errno as JSON, 0 for a connection. +import json +import socket + +targets = { + "metadata": ("169.254.169.254", 80), + "control_plane": ("203.0.113.10", 6443), + "outside_allowed_ips": ("203.0.113.10", 8080), +} +result = {} +for name, target in targets.items(): + try: + with socket.create_connection(target, timeout=10): + result[name] = 0 + except OSError as error: + result[name] = error.errno +print(json.dumps(result, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/listener_lifetime_probe.py b/tests/suites/conformance/src/scripts/listener_lifetime_probe.py new file mode 100644 index 0000000000..e53b2a6875 --- /dev/null +++ b/tests/suites/conformance/src/scripts/listener_lifetime_probe.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Closes listeners and exits a child that owns one, and checks each time that +# the port refuses connections and can be bound again. Prints the child +# listener's address and socket inode as JSON. +import errno, json, os, socket, subprocess, sys + +def assert_closed(address, inode): + # Check before bind can trigger the broker's stale-descriptor collection. + for table in ('/proc/net/tcp', '/proc/net/tcp6'): + for line in open(table): + fields = line.split() + assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(inode)), line + with socket.socket() as client: + client.settimeout(2) + assert client.connect_ex(tuple(address)) == errno.ECONNREFUSED, address + with socket.socket() as replacement: + replacement.bind(tuple(address)) + +def inode(sock): + return int(os.readlink('/proc/self/fd/' + str(sock.fileno()))[8:-1]) + +with socket.socket() as bound: + bound.bind(('127.0.0.1', 0)) + address = bound.getsockname() +with socket.socket() as client: + client.settimeout(2) + assert client.connect_ex(address) == errno.ECONNREFUSED + +for host in ('127.0.0.1', '::1'): + family = socket.AF_INET if host == '127.0.0.1' else socket.AF_INET6 + listener = socket.socket(family) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((host, 0)) + listener.listen(1) + listener.listen(8) + address, identity = listener.getsockname(), inode(listener) + for operation, expected in ((lambda: listener.bind(address), errno.EINVAL), + (lambda: listener.connect(address), errno.EISCONN)): + try: + operation() + raise AssertionError('operation on listener unexpectedly succeeded') + except OSError as error: + assert error.errno == expected, error + alias = listener.dup() + alias.settimeout(2) + listener.close() + with socket.socket(family) as client: + client.settimeout(2) + client.connect(address) + accepted, _ = alias.accept() + with accepted: + accepted.sendall(b'ok') + assert client.recv(2) == b'ok' + alias.close() + table = '/proc/net/tcp' if family == socket.AF_INET else '/proc/net/tcp6' + for line in open(table): + fields = line.split() + assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(identity)), line + with socket.socket(family) as client: + client.settimeout(2) + assert client.connect_ex(address) == errno.ECONNREFUSED + with socket.socket(family) as replacement: + replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + replacement.bind(address) + +# The child exits while still owning its listener. Its broker must not keep it alive. +child = r''' +import json, os, socket +s = socket.socket() +s.bind(('127.0.0.1', 0)) +s.listen() +inode = int(os.readlink('/proc/self/fd/' + str(s.fileno()))[8:-1]) +print(json.dumps([s.getsockname(), inode]), flush=True) +os._exit(0) +''' +result = subprocess.run([sys.executable, '-c', child], check=True, capture_output=True, text=True, timeout=10) +address, identity = json.loads(result.stdout) +assert_closed(address, identity) +print(json.dumps([address, identity])) diff --git a/tests/suites/conformance/src/scripts/localhost_client.py b/tests/suites/conformance/src/scripts/localhost_client.py new file mode 100644 index 0000000000..3621138742 --- /dev/null +++ b/tests/suites/conformance/src/scripts/localhost_client.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Serves HTTP on 127.0.0.1 and fetches from it, with no proxy variables in +# the environment. +import json +import os +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): + assert name not in os.environ, f'unexpected proxy environment variable: {name}' + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(b'{"message":"hello"}') + +server = HTTPServer(('127.0.0.1', 0), Handler) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() + +try: + with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: + print(json.dumps({ + 'proxy_env_absent': True, + 'payload': json.loads(response.read().decode()), + }), flush=True) +finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() diff --git a/tests/suites/conformance/src/scripts/preconnect_owner.py b/tests/suites/conformance/src/scripts/preconnect_owner.py new file mode 100644 index 0000000000..1b17222d69 --- /dev/null +++ b/tests/suites/conformance/src/scripts/preconnect_owner.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Opens an inheritable socket, writes its inode to /sandbox/preconnect-inode, +# and sleeps. It asks libc for the socket, since Python's socket constructor +# requests SOCK_CLOEXEC, which would hide an inheritance bug. +import ctypes +import os +import socket +import time + +libc = ctypes.CDLL(None, use_errno=True) +fd = libc.socket(socket.AF_INET, socket.SOCK_STREAM, 0) +assert fd >= 0, ctypes.get_errno() +assert os.get_inheritable(fd) +inode = int(os.readlink('/proc/self/fd/' + str(fd))[8:-1]) +with open('/sandbox/preconnect-inode.tmp', 'w') as f: + f.write(str(inode)) +os.rename('/sandbox/preconnect-inode.tmp', '/sandbox/preconnect-inode') +while True: + time.sleep(60) diff --git a/tests/suites/conformance/src/scripts/socket_not_inherited.py b/tests/suites/conformance/src/scripts/socket_not_inherited.py new file mode 100644 index 0000000000..488a0d2683 --- /dev/null +++ b/tests/suites/conformance/src/scripts/socket_not_inherited.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Fails if this process holds the socket whose inode is argv[1]. +import os +import sys + +inode = int(sys.argv[1]) +for fd in os.listdir('/proc/self/fd'): + try: + target = os.readlink('/proc/self/fd/' + fd) + except FileNotFoundError: + continue + assert target != 'socket:[' + str(inode) + ']', (fd, target) diff --git a/tests/suites/conformance/src/support.rs b/tests/suites/conformance/src/support.rs new file mode 100644 index 0000000000..dd512371df --- /dev/null +++ b/tests/suites/conformance/src/support.rs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Helpers the network suites share. Not a suite. +//! +//! Their clients are Python scripts in `src/scripts/` that take the server's +//! host and port as arguments, run in a long-running sandbox from +//! [`testkit::python_image`], and print their result as one JSON line. + +use serde_json::Value; +use testkit::sdk::raw::proto; +use testkit::{Context, ExecOutput, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; + +/// The name sandboxes reach the test host's servers by. +pub const TEST_HOST: &str = "host.openshell.internal"; + +/// A long-running sandbox from the Python image with `policy`, Ready. +pub async fn python_sandbox( + cx: &Context, + suffix: &str, + policy: proto::SandboxPolicy, +) -> Result { + cx.sandbox(suffix) + .policy(policy) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .create_ready() + .await +} + +/// Run `script` with the image's Python and `args`, and fail unless it +/// exits 0. +pub async fn run_python(sandbox: &Sandbox, script: &str, args: &[&str]) -> Result { + let mut command = vec!["python3", "-c", script]; + command.extend_from_slice(args); + sandbox + .exec(command) + .await? + .success() + .context("run the client in the sandbox") +} + +/// The last line of `output`'s stdout that parses as JSON. +pub fn json_result(output: &ExecOutput) -> Result { + output + .stdout + .lines() + .rev() + .find_map(|line| serde_json::from_str(line.trim()).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!( + "no JSON result in the client's output:\n{}\n{}", + output.stdout, output.stderr + )) + }) +} + +/// Fail unless each key in `expected` has its value in `result`. +pub fn expect_values>(result: &Value, expected: &[(K, Value)]) -> Result { + let wrong: Vec = expected + .iter() + .filter(|(key, value)| result[key.as_ref()] != *value) + .map(|(key, value)| { + let key = key.as_ref(); + format!("{key}: expected {value}, got {}", result[key]) + }) + .collect(); + testkit::ensure!(wrong.is_empty(), "{}\nresult: {result}", wrong.join("\n")); + Ok(()) +} + +/// `binaries:` for policy YAML that lets the image's Python connect. +pub fn python_binaries_yaml() -> String { + let paths: Vec = PYTHON_BINARIES + .iter() + .map(|path| format!("{{path: \"{path}\"}}")) + .collect(); + format!("[{}]", paths.join(", ")) +} From 7c76808eb3635ac743b4cd60216bc6d1dac6d66d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 15/63] test(conformance): port REST, GraphQL, JSON-RPC, and MCP L7 policy tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/network_l7.rs | 274 ++++++++++++++++++ .../suites/conformance/src/network_l7/mcp.rs | 221 ++++++++++++++ .../conformance/src/scripts/graphql_client.py | 242 ++++++++++++++++ .../src/scripts/jsonrpc_audit_client.py | 34 +++ .../conformance/src/scripts/jsonrpc_client.py | 197 +++++++++++++ .../conformance/src/scripts/mcp_client.py | 113 ++++++++ .../conformance/src/scripts/rest_client.py | 25 ++ .../src/scripts/rest_upgrade_client.py | 28 ++ 9 files changed, 1135 insertions(+) create mode 100644 tests/suites/conformance/src/network_l7.rs create mode 100644 tests/suites/conformance/src/network_l7/mcp.rs create mode 100644 tests/suites/conformance/src/scripts/graphql_client.py create mode 100644 tests/suites/conformance/src/scripts/jsonrpc_audit_client.py create mode 100644 tests/suites/conformance/src/scripts/jsonrpc_client.py create mode 100644 tests/suites/conformance/src/scripts/mcp_client.py create mode 100644 tests/suites/conformance/src/scripts/rest_client.py create mode 100644 tests/suites/conformance/src/scripts/rest_upgrade_client.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index f01f71f9b5..48932c0629 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -19,6 +19,7 @@ mod image; mod labels; mod lifecycle; mod network; +mod network_l7; mod persistent_workspace; mod policy_advisor; mod process; diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs new file mode 100644 index 0000000000..e443eaf151 --- /dev/null +++ b/tests/suites/conformance/src/network_l7.rs @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! L7 network policy: REST, GraphQL, and JSON-RPC rules the sandbox enforces +//! on requests a workload sends straight to a server, without a proxy. +//! +//! Ported from the `forward_proxy_l7_bypass`, `forward_proxy_graphql_l7`, +//! and `forward_proxy_jsonrpc_l7` e2e tests. The upstream servers run on the +//! test host ([`host_server_support`]) and implement none of the protocols: +//! the sandbox parses and enforces them before forwarding, so a server that +//! answers is enough to tell an allowed request from a denied one. + +mod mcp; + +use host_server_support::{HttpRecorder, Request, Response}; +use serde_json::json; +use testkit::{Context, Result, ResultExt as _, policy}; + +use crate::support::{ + TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const REST_CLIENT: &str = include_str!("scripts/rest_client.py"); +const REST_UPGRADE_CLIENT: &str = include_str!("scripts/rest_upgrade_client.py"); +const GRAPHQL_CLIENT: &str = include_str!("scripts/graphql_client.py"); +const JSONRPC_CLIENT: &str = include_str!("scripts/jsonrpc_client.py"); +const JSONRPC_AUDIT_CLIENT: &str = include_str!("scripts/jsonrpc_audit_client.py"); + +/// A REST endpoint that allows only `GET /allowed` passes that request and +/// answers a `POST` to the same path with `403` before it reaches upstream. +#[testkit::test] +async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(rest_upstream).await?; + let sandbox = python_sandbox(cx, "rest", rest_policy(server.port, "enforce")?).await?; + let output = run_python(&sandbox, REST_CLIENT, &[TEST_HOST, &port(&server)]).await?; + expect_values( + &json_result(&output)?, + &[("get", json!(200)), ("post", json!(403))], + )?; + let methods: Vec = server + .requests() + .iter() + .map(|request| request.method().to_string()) + .collect(); + testkit::ensure!( + methods == ["GET"], + "expected only the GET upstream; got {methods:?}" + ); + Ok(()) +} + +/// A permitted `GET` can't switch a REST endpoint into an uninspected +/// WebSocket tunnel, in enforce or audit mode: the sandbox answers the +/// upgrade with `403` and `unsupported_l7_protocol`. +#[testkit::test] +async fn rest_upgrade_is_refused_in_enforce_and_audit_modes(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(rest_upstream).await?; + for enforcement in ["enforce", "audit"] { + let sandbox = + python_sandbox(cx, enforcement, rest_policy(server.port, enforcement)?).await?; + run_python(&sandbox, REST_UPGRADE_CLIENT, &[TEST_HOST, &port(&server)]) + .await + .context(format!("{enforcement} mode"))? + .expect_stdout("REST_UPGRADE_REFUSED\n")?; + } + Ok(()) +} + +/// GraphQL rules allow listed query and mutation fields and registered +/// persisted queries, and deny unlisted fields, deny-rule mutations, +/// unregistered persisted queries, and a GET with a second, denied query. +/// Bodies sent with Content-Length and chunked both work, through urllib +/// and raw sockets. +#[testkit::test] +async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|request| match request.method() { + "GET" if request.path() == "/" || request.path().starts_with("/graphql?") => ok_json(), + "GET" => ok_json().status(418), + _ => ok_json(), + }) + .await?; + let policy = policy::base_with(&format!( + r"network_policies: + graphql: + name: graphql + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: graphql + enforcement: enforce + persisted_queries: allow_registered + graphql_persisted_queries: + abc123: + operation_type: query + operation_name: Viewer + fields: [viewer] + rules: + - allow: + operation_type: query + fields: [viewer] + - allow: + operation_type: mutation + fields: [createIssue] + deny_rules: + - operation_type: mutation + fields: [deleteRepository] + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "graphql", policy).await?; + let output = run_python(&sandbox, GRAPHQL_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let mut expected = Vec::new(); + for path in ["forward", "raw"] { + for (key, status) in [ + ("query_allowed", 200), + ("get_query_allowed", 200), + ("duplicate_get_denied", 403), + ("persisted_get_allowed", 200), + ("unregistered_persisted_get_denied", 403), + ("chunked_query_allowed", 200), + ("unlisted_field_denied", 403), + ("mutation_allowed", 200), + ("deny_rule_denied", 403), + ] { + expected.push((format!("{path}_{key}"), json!(status))); + } + } + expect_values(&json_result(&output)?, &expected) +} + +/// JSON-RPC rules allow listed methods whatever their params, deny a +/// deny-rule method, a batch with one denied call, and invalid JSON, through +/// urllib and raw sockets. +#[testkit::test] +async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(jsonrpc_upstream).await?; + let policy = policy::base_with(&format!( + r"network_policies: + jsonrpc: + name: jsonrpc + endpoints: + - host: {TEST_HOST} + port: {port} + path: /rpc + protocol: json-rpc + enforcement: enforce + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + - allow: + method: tools/list + - allow: + method: tools/call + deny_rules: + - method: tools/delete + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "jsonrpc", policy).await?; + let output = run_python(&sandbox, JSONRPC_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let mut expected = Vec::new(); + for path in ["forward", "raw"] { + for (key, status) in [ + ("method_initialize_allowed", 200), + ("method_tools_list_allowed", 200), + ("method_tools_call_allowed", 200), + ("method_tools_call_with_unmatched_params_allowed", 200), + ("method_tools_delete_denied", 403), + ] { + expected.push((format!("{path}_{key}"), json!(status))); + } + } + for (key, status) in [ + ("forward_batch_all_allowed", 200), + ("forward_batch_one_denied", 403), + ("forward_invalid_json_denied", 403), + ] { + expected.push((key.to_string(), json!(status))); + } + expect_values(&json_result(&output)?, &expected) +} + +/// A JSON-RPC endpoint without `enforcement` audits an unlisted method and +/// passes it, but still denies a response frame sent as a request. +#[testkit::test] +async fn jsonrpc_default_audit_mode_still_denies_response_frames( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::serve(jsonrpc_upstream).await?; + let policy = policy::base_with(&format!( + r"network_policies: + jsonrpc_audit: + name: jsonrpc_audit + endpoints: + - host: {TEST_HOST} + port: {port} + path: /rpc + protocol: json-rpc + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "jsonaudit", policy).await?; + let output = run_python(&sandbox, JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]).await?; + expect_values( + &json_result(&output)?, + &[ + ("forward_unknown_method_audited", json!(200)), + ("forward_response_frame_hard_denied", json!(403)), + ], + ) +} + +/// A REST endpoint on `port` in `enforcement` mode that allows only +/// `GET /allowed`. +fn rest_policy(port: u16, enforcement: &str) -> Result { + policy::base_with(&format!( + r"network_policies: + rest: + name: rest + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + enforcement: {enforcement} + rules: + - allow: + method: GET + path: /allowed + binaries: {binaries} +", + binaries = python_binaries_yaml(), + )) +} + +/// Answers a WebSocket upgrade with `101`, and `GET` or `DELETE` with +/// `200`, so that a refused upgrade shows the sandbox refused it. +fn rest_upstream(request: &Request) -> Response { + match request.method() { + "GET" if request.header("upgrade").is_some() => Response::new(101) + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .header("Sec-WebSocket-Accept", "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + "GET" | "DELETE" => ok_json(), + _ => Response::new(405), + } +} + +fn jsonrpc_upstream(request: &Request) -> Response { + match request.method() { + "POST" => Response::json(&json!({"jsonrpc": "2.0", "id": 1, "result": {}})), + _ => Response::new(200), + } +} + +fn ok_json() -> Response { + Response::json(&json!({"ok": true})) +} + +fn port(server: &HttpRecorder) -> String { + server.port.to_string() +} diff --git a/tests/suites/conformance/src/network_l7/mcp.rs b/tests/suites/conformance/src/network_l7/mcp.rs new file mode 100644 index 0000000000..3ce7742b1e --- /dev/null +++ b/tests/suites/conformance/src/network_l7/mcp.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! MCP request profiles: the sandbox authorizes tool calls by MCP revision, +//! for the sessionless revision, legacy initialization, and an endpoint that +//! accepts several revisions. +//! +//! Ported from the `mcp_sessionless` e2e test. The fixture on the test host +//! serves initialization, discovery, tools, and a bounded subscription +//! stream, and reports the requests it received in each tool result, so the +//! client can tell a policy denial from a request the fixture accepted. + +use std::sync::{Arc, Mutex, PoisonError}; + +use host_server_support::{HttpRecorder, Request, Response}; +use serde_json::{Value, json}; +use testkit::{Context, Result, ResultExt as _, policy}; + +use crate::support::{TEST_HOST, python_binaries_yaml, python_sandbox, run_python}; + +const MCP_CLIENT: &str = include_str!("../scripts/mcp_client.py"); +const SESSIONLESS: &str = "2026-07-28"; + +/// The sessionless revision: discovery, an allowed and a denied tool, and a +/// subscription stream, with the request metadata the revision requires. +#[testkit::test] +async fn mcp_sessionless_discovery_tools_and_subscription(cx: &mut Context) -> testkit::Result { + run_profile(cx, "mcpsl", &[SESSIONLESS]).await +} + +/// Each legacy revision: initialization, then an allowed and a denied tool. +#[testkit::test] +async fn mcp_legacy_revisions_authorize_tools(cx: &mut Context) -> testkit::Result { + for (suffix, version) in [("mcp0326", "2025-03-26"), ("mcp0618", "2025-06-18")] { + run_profile(cx, suffix, &[version]).await?; + } + Ok(()) +} + +/// An endpoint that accepts a legacy and the sessionless revision authorizes +/// tools under each, against one fixture. +#[testkit::test] +async fn mcp_multi_version_endpoint_authorizes_tools(cx: &mut Context) -> testkit::Result { + run_profile(cx, "mcpmv", &["2025-11-25", SESSIONLESS]).await +} + +/// Start a fresh fixture for `versions`, and run [`MCP_CLIENT`] for them in +/// a sandbox whose policy accepts them. +async fn run_profile(cx: &Context, suffix: &str, versions: &[&str]) -> Result { + let supported: Vec = versions.iter().map(ToString::to_string).collect(); + let received = Arc::new(Mutex::new(Vec::new())); + let server = + HttpRecorder::serve(move |request| fixture(request, &supported, &received)).await?; + + let legacy_rules = if versions.iter().any(|version| *version != SESSIONLESS) { + " - allow:\n method: initialize\n - allow:\n method: notifications/initialized\n" + } else { + "" + }; + let versions_json = serde_json::to_string(versions).context("encode the MCP revisions")?; + let policy = policy::base_with(&format!( + r"network_policies: + mcp: + name: mcp + endpoints: + - host: {TEST_HOST} + port: {port} + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + versions: {versions_json} + max_body_bytes: 65536 + rules: +{legacy_rules} - allow: + method: server/discover + - allow: + method: tools/call + tool: read_status + - allow: + method: subscriptions/listen + deny_rules: + - method: tools/call + tool: read_details + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, suffix, policy).await?; + let output = run_python( + &sandbox, + MCP_CLIENT, + &[TEST_HOST, &server.port.to_string(), &versions_json], + ) + .await + .context(format!("MCP revisions {versions:?}"))?; + for version in versions { + let marker = format!("MCP_OK version={version}"); + testkit::ensure!( + output.stdout.lines().any(|line| line == marker), + "{versions:?}: the client didn't finish {version}:\n{}", + output.stdout + ); + } + Ok(()) +} + +/// The MCP fixture. It records each parsed request as `[version, method, +/// tool]` in `received` before checking it, so rejected requests stay +/// visible in later tool results. +fn fixture(request: &Request, supported: &[String], received: &Mutex>) -> Response { + if request.method() == "GET" { + return Response::new(if request.path() == "/" { 200 } else { 405 }); + } + let Ok(message) = serde_json::from_str::(&request.body) else { + return bad_request("the request body is not JSON"); + }; + let method = message["method"].as_str().unwrap_or_default(); + let params = &message["params"]; + let version = if method == "initialize" { + params["protocolVersion"].as_str().map(ToString::to_string) + } else { + request.header("mcp-protocol-version") + }; + let received_now = { + let mut received = received.lock().unwrap_or_else(PoisonError::into_inner); + received.push(json!([version, method, params.get("name")])); + received.clone() + }; + let Some(version) = version.filter(|version| supported.contains(version)) else { + return bad_request("request revision did not reach the fixture intact"); + }; + if request.path() != "/mcp" { + return bad_request("request revision did not reach the fixture intact"); + } + let sessionless = version == SESSIONLESS; + if sessionless { + let meta = ¶ms["_meta"]; + let intact = request.header("mcp-protocol-version").as_deref() == Some(&version) + && request.header("mcp-method").as_deref() == Some(method) + && meta["io.modelcontextprotocol/protocolVersion"] == version.as_str() + && meta["io.modelcontextprotocol/clientCapabilities"] == json!({}) + && (method != "tools/call" + || request.header("mcp-name").as_deref() == params["name"].as_str()); + if !intact { + return bad_request("request metadata did not reach the fixture intact"); + } + } + + let result = match method { + "initialize" if !sessionless => json!({ + "protocolVersion": version, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openshell-profile-fixture", "version": "1"}, + }), + "notifications/initialized" if !sessionless => return Response::new(202), + "server/discover" if sessionless => json!({ + "supportedVersions": [version], + "capabilities": {"tools": {"listChanged": true}}, + "ttlMs": 0, + "cacheScope": "private", + "_meta": { + "io.modelcontextprotocol/serverInfo": { + "name": "openshell-sessionless-fixture", "version": "1" + } + }, + }), + // Both tool names work upstream; the sandbox owns the policy denial. + "tools/call" => { + let mut result = json!({ + "content": [{"type": "text", "text": params["name"]}], + "isError": false, + "_meta": {"fixtureRequests": received_now}, + }); + if sessionless { + result["resultType"] = json!("complete"); + } + result + } + "subscriptions/listen" if sessionless => return subscription(&message), + _ => return bad_request("unexpected method for the selected fixture revision"), + }; + Response::json(&json!({"jsonrpc": "2.0", "id": message["id"], "result": result})) +} + +/// A subscription stream that acknowledges, sends one notification, and +/// completes. +fn subscription(message: &Value) -> Response { + let meta = json!({"io.modelcontextprotocol/subscriptionId": message["id"]}); + let events = [ + json!({ + "jsonrpc": "2.0", + "method": "notifications/subscriptions/acknowledged", + "params": {"notifications": message["params"]["notifications"], "_meta": meta}, + }), + json!({ + "jsonrpc": "2.0", + "method": "notifications/tools/list_changed", + "params": {"_meta": meta}, + }), + json!({ + "jsonrpc": "2.0", + "id": message["id"], + "result": {"resultType": "complete", "_meta": meta}, + }), + ]; + let body: String = events + .iter() + .map(|event| format!("event: message\ndata: {event}\n\n")) + .collect(); + Response::new(200) + .header("Content-Type", "text/event-stream") + .body(body) +} + +fn bad_request(reason: &str) -> Response { + Response::new(400) + .header("Content-Type", "text/plain") + .body(reason) +} diff --git a/tests/suites/conformance/src/scripts/graphql_client.py b/tests/suites/conformance/src/scripts/graphql_client.py new file mode 100644 index 0000000000..f8d4c07c6a --- /dev/null +++ b/tests/suites/conformance/src/scripts/graphql_client.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends GraphQL queries, mutations, and persisted queries to HOST:PORT with +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as +# JSON. +import json +import socket +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +DETAILS = {} + +QUERY_VIEWER = "query Viewer { viewer { login } }" +QUERY_REPOSITORY = "query Repo { repository(owner:\"o\", name:\"r\") { id } }" +MUTATION_CREATE = "mutation Create { createIssue(input:{repositoryId:\"r\", title:\"t\", body:\"b\"}) { issue { id } } }" +MUTATION_DELETE = "mutation Delete { deleteRepository(input:{repositoryId:\"r\"}) { clientMutationId } }" + +def forward_status(query): + body = json.dumps({"query": query}).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_get_status(query): + encoded = urllib.parse.urlencode({"query": query}) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?{encoded}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_duplicate_get_status(): + safe = urllib.parse.quote_plus(QUERY_VIEWER) + unsafe = urllib.parse.quote_plus(MUTATION_DELETE) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?query={safe}&query={unsafe}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_persisted_get_status(hash_value): + extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) + encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?{encoded}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_chunked_status(query): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + + with socket.create_connection((HOST, PORT), timeout=15) as sock: + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Transfer-Encoding: chunked\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + chunk + sock.sendall(request) + response, body = read_response(sock) + DETAILS["forward_chunked_query_allowed_detail"] = body.decode(errors="replace") + return status_code(response, "forward_chunked_response") + +def read_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def read_response(sock): + response = read_until(sock, b"\r\n\r\n") + headers, _, body = response.partition(b"\r\n\r\n") + content_length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + content_length = int(line.split(b":", 1)[1].strip()) + break + while len(body) < content_length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + return response, body + +def status_code(response, label): + parts = response.split() + if len(parts) < 2: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") + try: + return int(parts[1]) + except ValueError as error: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error + +def raw_http_status(label, request): + last_error = None + for attempt in range(5): + try: + with socket.create_connection((HOST, PORT), timeout=15) as sock: + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response = read_until(sock, b"\r\n\r\n") + return status_code(response, f"{label}_response") + except (OSError, RuntimeError) as error: + last_error = error + DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) + time.sleep(0.2) + + raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") + +def raw_status(query, label): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + body + return raw_http_status(label, request) + +def raw_get_status(query, label): + target = f"{HOST}:{PORT}" + encoded = urllib.parse.urlencode({"query": query}) + + request = ( + f"GET /graphql?{encoded} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status(label, request) + +def raw_duplicate_get_status(): + target = f"{HOST}:{PORT}" + safe = urllib.parse.quote_plus(QUERY_VIEWER) + unsafe = urllib.parse.quote_plus(MUTATION_DELETE) + + request = ( + f"GET /graphql?query={safe}&query={unsafe} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status("raw_duplicate_get_denied", request) + +def raw_persisted_get_status(hash_value, label): + target = f"{HOST}:{PORT}" + extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) + encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) + + request = ( + f"GET /graphql?{encoded} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status(label, request) + +def raw_chunked_status(query): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Transfer-Encoding: chunked\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + chunk + return raw_http_status("raw_chunked_query_allowed", request) + +results = { + "forward_query_allowed": forward_status(QUERY_VIEWER), + "forward_get_query_allowed": forward_get_status(QUERY_VIEWER), + "forward_duplicate_get_denied": forward_duplicate_get_status(), + "forward_persisted_get_allowed": forward_persisted_get_status("abc123"), + "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"), + "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER), + "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), + "forward_mutation_allowed": forward_status(MUTATION_CREATE), + "forward_deny_rule_denied": forward_status(MUTATION_DELETE), + "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), + "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), + "raw_duplicate_get_denied": raw_duplicate_get_status(), + "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), + "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), + "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), + "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), + "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), + "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), +} +results.update(DETAILS) +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py b/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py new file mode 100644 index 0000000000..c1145be040 --- /dev/null +++ b/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends an unlisted JSON-RPC method and a response frame to HOST:PORT/rpc, +# and prints each status as JSON. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + +def post_jsonrpc(body): + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +results = { + "forward_unknown_method_audited": post_jsonrpc({"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}), + "forward_response_frame_hard_denied": post_jsonrpc({"jsonrpc": "2.0", "id": 1, "result": {}}), +} +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/jsonrpc_client.py b/tests/suites/conformance/src/scripts/jsonrpc_client.py new file mode 100644 index 0000000000..919009584f --- /dev/null +++ b/tests/suites/conformance/src/scripts/jsonrpc_client.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends JSON-RPC calls, batches, and invalid JSON to HOST:PORT/rpc with +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as +# JSON. +import json +import socket +import sys +import time +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +DETAILS = { + "debug_target": {"host": HOST, "port": PORT}, +} + +def text(data): + return data.decode(errors="replace") + +def selected_headers(headers): + return { + key.lower(): value + for key, value in headers.items() + if key.lower() in ("content-type", "content-length", "server") + } + +def record_http_error(label, error, request_body): + response_body = error.read() + DETAILS[f"{label}_request"] = request_body + DETAILS[f"{label}_response"] = { + "status": error.code, + "reason": str(error.reason), + "headers": selected_headers(error.headers), + "body": text(response_body), + } + return error.code + +def post_jsonrpc(label, method, params=None, req_id=1): + body = {"jsonrpc": "2.0", "id": req_id, "method": method} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, body) + +def post_jsonrpc_batch(label, requests): + encoded = json.dumps(requests).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, requests) + +def post_invalid_json(label): + encoded = b"not valid json {" + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json", "Content-Length": str(len(encoded))}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, text(encoded)) + +def read_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def read_response(sock): + response = read_until(sock, b"\r\n\r\n") + headers, _, body = response.partition(b"\r\n\r\n") + content_length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + content_length = int(line.split(b":", 1)[1].strip()) + break + while len(body) < content_length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + return response, body + +def status_code(response, label): + parts = response.split() + if len(parts) < 2: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") + try: + return int(parts[1]) + except ValueError as error: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error + +def record_raw_response(label, response, body=b""): + code = status_code(response, label) + if code != 200: + DETAILS[f"{label}_raw"] = text(response) + if body: + DETAILS[f"{label}_body"] = text(body) + return code + +def raw_http_status(label, request): + last_error = None + for attempt in range(5): + try: + with socket.create_connection((HOST, PORT), timeout=15) as sock: + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response, body = read_response(sock) + return record_raw_response(f"{label}_response", response, body) + except (OSError, RuntimeError) as error: + last_error = error + DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) + time.sleep(0.2) + + raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") + +def raw_jsonrpc_status(method, params, label): + target = f"{HOST}:{PORT}" + body = {"jsonrpc": "2.0", "id": 1, "method": method} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = ( + f"POST /rpc HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Content-Length: {len(encoded)}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + encoded + return raw_http_status(label, request) + +results = { + # forward proxy — method-only allow rules + "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}), + "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), + + # forward proxy — method allow/deny rules + "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {"name": "read_status"}), + "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}), + "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {"name": "purge_cache"}), + + # forward proxy — batch: all requests allowed + "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "read_status"}}, + ]), + + # forward proxy — batch: one denied request causes full batch denial + "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {"name": "purge_cache"}}, + ]), + + # forward proxy — invalid JSON body fails closed before generic rules apply + "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), + + # raw socket path — representative allowed and denied cases + "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}, "raw_method_initialize_allowed"), + "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), + "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {"name": "read_status"}, "raw_method_tools_call_allowed"), + "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}, "raw_method_tools_call_with_unmatched_params_allowed"), + "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {"name": "purge_cache"}, "raw_method_tools_delete_denied"), +} +results.update(DETAILS) +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/mcp_client.py b/tests/suites/conformance/src/scripts/mcp_client.py new file mode 100644 index 0000000000..7845555f19 --- /dev/null +++ b/tests/suites/conformance/src/scripts/mcp_client.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs an MCP client against the fixture at HOST:PORT for each revision in the +# JSON list VERSIONS: discovery or initialization, an allowed and a denied +# tool, and for the sessionless revision, a subscription. Prints +# `MCP_OK version=` after each revision's checks pass. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +VERSIONS = json.loads(sys.argv[3]) +SESSIONLESS = "2026-07-28" + +# Direct client connections pass through the sandbox's transparent interception. +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def post(request_id, method, params, version): + params = dict(params) + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if method != "initialize": + headers["MCP-Protocol-Version"] = version + if version == SESSIONLESS: + params["_meta"] = { + "io.modelcontextprotocol/protocolVersion": version, + "io.modelcontextprotocol/clientCapabilities": {}, + "io.modelcontextprotocol/clientInfo": {"name": "openshell-e2e", "version": "1"}, + } + headers["Mcp-Method"] = method + if method == "tools/call": + headers["Mcp-Name"] = params["name"] + message = {"jsonrpc": "2.0", "method": method, "params": params} + if request_id is not None: + message["id"] = request_id + request = urllib.request.Request( + f"http://{HOST}:{PORT}/mcp", + data=json.dumps(message).encode(), + headers=headers, + method="POST", + ) + try: + with opener.open(request, timeout=15) as response: + return response.status, response.headers.get_content_type(), response.read() + except urllib.error.HTTPError as error: + return error.code, error.headers.get_content_type(), error.read() + + +expected_receipts = [] +for version in VERSIONS: + if version == SESSIONLESS: + status, content_type, body = post(1, "server/discover", {}, version) + assert status == 200, (version, "discovery", status, body) + assert content_type == "application/json", (version, content_type) + discovery = json.loads(body) + assert discovery["id"] == 1, discovery + assert discovery["result"]["supportedVersions"] == [version], discovery + expected_receipts.append([version, "server/discover", None]) + else: + status, _, body = post(1, "initialize", { + "protocolVersion": version, + "capabilities": {}, + "clientInfo": {"name": "openshell-e2e", "version": "1"}, + }, version) + assert status == 200, (version, "initialize", status, body) + assert json.loads(body)["result"]["protocolVersion"] == version, body + expected_receipts.append([version, "initialize", None]) + status, _, body = post(None, "notifications/initialized", {}, version) + assert status == 202, (version, "initialized", status, body) + expected_receipts.append([version, "notifications/initialized", None]) + + status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, version) + assert status == 200, (version, "allowed tool", status, body) + tool = json.loads(body) + assert tool["id"] == 2, tool + assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool + expected_receipts.append([version, "tools/call", "read_status"]) + assert tool["result"]["_meta"]["fixtureRequests"] == expected_receipts, tool + + status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, version) + assert status == 403, (version, "denied tool", status, body) + + # The fixture accepts both tools. A later allowed call proves the denial + # came from the proxy and no denied operation reached the upstream. + status, _, body = post(4, "tools/call", {"name": "read_status", "arguments": {}}, version) + assert status == 200, (version, "receipt tool", status, body) + receipt = json.loads(body) + assert receipt["id"] == 4, receipt + expected_receipts.append([version, "tools/call", "read_status"]) + assert receipt["result"]["_meta"]["fixtureRequests"] == expected_receipts, receipt + + if version == SESSIONLESS: + status, content_type, body = post( + 5, "subscriptions/listen", {"notifications": {"toolsListChanged": True}}, version + ) + assert status == 200, (version, "subscription", status, body) + assert content_type == "text/event-stream", (content_type, body) + events = [json.loads(line[6:]) for line in body.decode().splitlines() if line.startswith("data: ")] + assert len(events) == 3, events + assert events[0]["method"] == "notifications/subscriptions/acknowledged", events + assert events[0]["params"]["notifications"] == {"toolsListChanged": True}, events + assert events[1]["method"] == "notifications/tools/list_changed", events + assert events[1]["params"]["_meta"]["io.modelcontextprotocol/subscriptionId"] == 5, events + assert events[2]["id"] == 5 and events[2]["result"]["resultType"] == "complete", events + expected_receipts.append([version, "subscriptions/listen", None]) + + print(f"MCP_OK version={version}", flush=True) diff --git a/tests/suites/conformance/src/scripts/rest_client.py b/tests/suites/conformance/src/scripts/rest_client.py new file mode 100644 index 0000000000..d59543a760 --- /dev/null +++ b/tests/suites/conformance/src/scripts/rest_client.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Prints the status of GET and POST to /allowed as JSON. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + + +def status(method, data=None): + request = urllib.request.Request(f"http://{HOST}:{PORT}/allowed", data=data, method=method) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + + +print(json.dumps({"get": status("GET"), "post": status("POST", b"test")}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/rest_upgrade_client.py b/tests/suites/conformance/src/scripts/rest_upgrade_client.py new file mode 100644 index 0000000000..305742fe46 --- /dev/null +++ b/tests/suites/conformance/src/scripts/rest_upgrade_client.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Asks to upgrade a REST endpoint to WebSocket with an origin-form and an +# absolute-form target, and fails unless the sandbox refuses both. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + +for target in ["/allowed", f"http://{HOST}:{PORT}/allowed"]: + s = socket.create_connection((HOST, PORT), timeout=15) + request = ("GET " + target + f" HTTP/1.1\r\nHost: {HOST}:{PORT}\r\n" + "Connection: Upgrade\r\nUpgrade: websocket\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n") + s.sendall(request.encode()) + response = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + response += chunk + s.close() + assert response.startswith(b"HTTP/1.1 403"), response + assert b"unsupported_l7_protocol" in response, response + assert b"REST endpoints" in response, response +print("REST_UPGRADE_REFUSED") From 9b307bad3e3d0af316219ebfa92869483672dbef Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 16/63] test(conformance): port transparent egress pipeline tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + .../suites/conformance/src/network_egress.rs | 439 ++++++++++++++++++ .../conformance/src/scripts/echo_client.py | 23 + .../src/scripts/pipeline_client.py | 28 ++ .../src/scripts/redaction_client.py | 53 +++ .../src/scripts/reload_keep_alive_client.py | 66 +++ .../conformance/src/scripts/status_client.py | 44 ++ .../src/scripts/uninspectable_client.py | 30 ++ 8 files changed, 684 insertions(+) create mode 100644 tests/suites/conformance/src/network_egress.rs create mode 100644 tests/suites/conformance/src/scripts/echo_client.py create mode 100644 tests/suites/conformance/src/scripts/pipeline_client.py create mode 100644 tests/suites/conformance/src/scripts/redaction_client.py create mode 100644 tests/suites/conformance/src/scripts/reload_keep_alive_client.py create mode 100644 tests/suites/conformance/src/scripts/status_client.py create mode 100644 tests/suites/conformance/src/scripts/uninspectable_client.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 48932c0629..7c40b58bd2 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -19,6 +19,7 @@ mod image; mod labels; mod lifecycle; mod network; +mod network_egress; mod network_l7; mod persistent_workspace; mod policy_advisor; diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs new file mode 100644 index 0000000000..8a63120681 --- /dev/null +++ b/tests/suites/conformance/src/network_egress.rs @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transparent egress: what the supervisor does with connections a workload +//! makes straight to a destination, without proxy settings. Policy reloads +//! apply to open connections, `tls: skip` relays raw bytes, middleware +//! inspects or blocks traffic, and pipelined requests are authorized one at +//! a time. +//! +//! Ported from the `proxy_egress_pipeline` e2e tests. The upstream servers +//! run on the test host ([`host_server_support`]). + +use std::time::Duration; + +use host_server_support::{HttpRecorder, Request, Response, TcpServer}; +use serde_json::{Value, json}; +use testkit::policy::{self, Endpoint as _}; +use testkit::sdk::raw::proto; +use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; + +use crate::support::{ + TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const STATUS_CLIENT: &str = include_str!("scripts/status_client.py"); +const RELOAD_KEEP_ALIVE_CLIENT: &str = include_str!("scripts/reload_keep_alive_client.py"); +const ECHO_CLIENT: &str = include_str!("scripts/echo_client.py"); +const UNINSPECTABLE_CLIENT: &str = include_str!("scripts/uninspectable_client.py"); +const REDACTION_CLIENT: &str = include_str!("scripts/redaction_client.py"); +const PIPELINE_CLIENT: &str = include_str!("scripts/pipeline_client.py"); + +/// Files [`RELOAD_KEEP_ALIVE_CLIENT`] signals through. +const RELOAD_READY: &str = "/tmp/proxy-reload-ready"; +const RELOAD_GO: &str = "/tmp/proxy-reload-go"; +const RELOAD_RESULT: &str = "/tmp/proxy-reload-result"; +const RELOAD_LOG: &str = "/tmp/proxy-reload-client.log"; + +/// The reply [`TcpServer::reply_when_idle`] gives a pipelined request, so the +/// connection stays usable for the next one. +const PIPELINE_REPLY: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"; + +/// A policy reload that removes access applies to new connections and to a +/// keep-alive connection opened before it: the next request on that +/// connection isn't forwarded. +#[testkit::test] +async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; + let allowed = allow_server(server.port); + let sandbox = python_sandbox(cx, "reload", allowed.clone()).await?; + policy::set(&sandbox, &allowed).await?; + + let port = server.port.to_string(); + sandbox + .exec([ + "sh", + "-c", + &format!("nohup python3 -c \"$1\" {TEST_HOST} {port} >{RELOAD_LOG} 2>&1 &"), + "reload-client", + RELOAD_KEEP_ALIVE_CLIENT, + ]) + .await? + .success() + .context("start the keep-alive client")?; + wait_for_file(&sandbox, RELOAD_READY).await?; + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("before the reload")?; + + policy::set(&sandbox, &policy::base()).await?; + sandbox + .exec(["touch", RELOAD_GO]) + .await? + .success() + .context("release the keep-alive client")?; + let result: Value = serde_json::from_str(wait_for_file(&sandbox, RELOAD_RESULT).await?.trim()) + .context("parse the keep-alive client's result")?; + testkit::ensure!( + result["failed_closed"] == true, + "the open connection forwarded a request after the reload: {result}" + ); + expect_statuses(&sandbox, &port, |status| status != 200) + .await + .context("after the reload") +} + +/// The gateway rejects a policy update with two endpoints for one host and +/// port, one of them `tls: skip`. The sandbox keeps its policy revisions and +/// still forwards requests under the active policy. +#[testkit::test] +async fn ambiguous_policy_update_is_rejected_and_active_policy_kept( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; + let allowed = allow_server(server.port); + let sandbox = python_sandbox(cx, "ambig", allowed.clone()).await?; + policy::set(&sandbox, &allowed).await?; + let port = server.port.to_string(); + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("before the rejected update")?; + let revisions = policy_revisions(cx, &sandbox).await?; + let connections = server.connections(); + + let mut ambiguous = policy::base(); + policy::allow( + &mut ambiguous, + "terminating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ); + policy::allow( + &mut ambiguous, + "passthrough", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .tls_skip(); + let file = testkit::YamlFile::new(&policy::to_yaml(&ambiguous)?)?; + let output = cx + .cli([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .failure()?; + testkit::ensure!( + output.report_text().contains("ambiguity validation failed"), + "the rejection doesn't explain the ambiguity:\n{}", + output.report_text() + ); + + let after = policy_revisions(cx, &sandbox).await?; + testkit::ensure!( + after == revisions, + "the rejected policy changed the revisions:\nbefore: {revisions}\nafter: {after}" + ); + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("after the rejected update")?; + testkit::ensure!( + server.connections() > connections, + "requests under the active policy didn't reach the server" + ); + Ok(()) +} + +/// An endpoint with `tls: skip` relays bytes that are neither HTTP nor TLS +/// unchanged in both directions. +#[testkit::test] +async fn tls_skip_relays_opaque_bytes(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let mut policy = policy::base(); + allow(&mut policy, server.port).tls_skip(); + let sandbox = python_sandbox(cx, "tlsskip", policy).await?; + let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]; + payload.extend_from_slice(b"not-http-or-tls"); + payload.extend(0..64_u8); + run_python( + &sandbox, + ECHO_CLIENT, + &[TEST_HOST, &server.port.to_string(), &hex(&payload)], + ) + .await? + .expect_stdout("ECHO_OK\n")?; + Ok(()) +} + +/// A redacting middleware rewrites an API key in each request body before +/// it reaches upstream. +#[testkit::test] +async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|request: &Request| { + Response::new(200) + .header("Content-Type", "application/json") + .body(request.body.clone()) + }) + .await?; + let policy = middleware_policy(server.port, "fail_closed", "")?; + let sandbox = python_sandbox(cx, "redact", policy).await?; + let output = run_python( + &sandbox, + REDACTION_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await?; + let result = json_result(&output)?; + for request in ["first", "second"] { + expect_values(&result[request], &[("api_key", json!("[REDACTED]"))]) + .context(format!("the {request} request"))?; + } + Ok(()) +} + +/// A `fail_closed` middleware blocks traffic it can't inspect before any +/// byte reaches upstream, and the sandbox logs why. +#[testkit::test] +async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let policy = middleware_policy(server.port, "fail_closed", "")?; + let sandbox = python_sandbox(cx, "failclosed", policy).await?; + run_python( + &sandbox, + UNINSPECTABLE_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout("BLOCKED\n")?; + // Give a forwarded payload time to arrive before checking it didn't. + tokio::time::sleep(Duration::from_millis(200)).await; + let received = server.received(); + testkit::ensure!( + received.is_empty(), + "the payload reached upstream: {:?}", + String::from_utf8_lossy(&received) + ); + sandbox + .logs_until("the middleware denial", |logs| { + logs.contains("openshell.middleware.traffic_uninspectable") + && logs.contains( + "Unsupported tunnel protocol cannot be inspected by required middleware", + ) + }) + .await?; + Ok(()) +} + +/// A `fail_open` middleware on a `tls: skip` endpoint lets traffic it can't +/// inspect through unchanged. +#[testkit::test] +async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let policy = middleware_policy(server.port, "fail_open", " tls: skip\n")?; + let sandbox = python_sandbox(cx, "failopen", policy).await?; + let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80]; + payload.extend_from_slice(b"middleware-bypass"); + run_python( + &sandbox, + ECHO_CLIENT, + &[TEST_HOST, &server.port.to_string(), &hex(&payload)], + ) + .await? + .expect_stdout("ECHO_OK\n")?; + let received = server.received(); + testkit::ensure!( + received == payload, + "upstream received {:?}; expected the payload unchanged", + String::from_utf8_lossy(&received) + ); + Ok(()) +} + +/// A denied request pipelined behind an allowed one is answered with `403` +/// and never reaches upstream, not even as extra bytes after the first +/// request. The relay strips the hop-by-hop `Connection` header. +#[testkit::test] +async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { + let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; + let sandbox = python_sandbox(cx, "pipeline", rest_policy(server.port, "GET")?).await?; + let target = format!("{TEST_HOST}:{}", server.port); + let requests = format!( + "GET /allowed HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n\ + POST /blocked HTTP/1.1\r\nHost: {target}\r\nContent-Length: 0\r\n\r\n" + ); + run_pipeline(&sandbox, server.port, &requests).await?; + let received = String::from_utf8_lossy(&server.received()).into_owned(); + testkit::ensure!( + received.starts_with("GET /allowed HTTP/1.1\r\n") + && !received.to_ascii_lowercase().contains("\r\nconnection:") + && !received.contains("/blocked"), + "expected only the allowed request upstream, without Connection; got:\n{received}" + ); + Ok(()) +} + +/// A denied request pipelined behind an allowed chunked request is +/// authorized on its own: upstream sees the chunked request whole and +/// nothing after it. +#[testkit::test] +async fn chunked_pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { + let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; + let sandbox = python_sandbox(cx, "chunked", rest_policy(server.port, "POST")?).await?; + let target = format!("{TEST_HOST}:{}", server.port); + let requests = format!( + "POST /allowed HTTP/1.1\r\nHost: {target}\r\nTransfer-Encoding: chunked\r\n\ + Connection: keep-alive\r\n\r\n0\r\n\r\n\ + DELETE /blocked HTTP/1.1\r\nHost: {target}\r\nContent-Length: 0\r\n\r\n" + ); + run_pipeline(&sandbox, server.port, &requests).await?; + let received = String::from_utf8_lossy(&server.received()).into_owned(); + testkit::ensure!( + received.starts_with("POST /allowed HTTP/1.1\r\n") + && received.ends_with("0\r\n\r\n") + && !received.contains("/blocked"), + "expected only the allowed chunked request upstream; got:\n{received}" + ); + Ok(()) +} + +/// [`policy::base`] with the image's Python allowed to reach `TEST_HOST` on +/// `port`. +fn allow_server(port: u16) -> proto::SandboxPolicy { + let mut policy = policy::base(); + allow(&mut policy, port); + policy +} + +fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { + policy::allow(policy, "egress", TEST_HOST, port, PYTHON_BINARIES) +} + +/// An endpoint on `port` with the `openshell/regex` redacting middleware, +/// `on_error`, and the extra endpoint YAML in `endpoint_options`. +fn middleware_policy( + port: u16, + on_error: &str, + endpoint_options: &str, +) -> Result { + policy::base_with(&format!( + r#"network_middlewares: + regex-redactor: + name: Redact API tokens + middleware: openshell/regex + order: 10 + config: + mode: redact + on_error: {on_error} + endpoints: + include: ["{TEST_HOST}"] + exclude: [] +network_policies: + egress: + name: egress + endpoints: + - host: {TEST_HOST} + port: {port} +{endpoint_options} binaries: {binaries} +"#, + binaries = python_binaries_yaml(), + )) +} + +/// A REST endpoint on `port` that allows only `method` on `/allowed`. +fn rest_policy(port: u16, method: &str) -> Result { + policy::base_with(&format!( + r"network_policies: + egress: + name: egress + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + enforcement: enforce + rules: + - allow: + method: {method} + path: /allowed + binaries: {binaries} +", + binaries = python_binaries_yaml(), + )) +} + +/// Send `requests` pipelined in one write, and fail unless the first gets +/// `200` and the second `403`. +async fn run_pipeline(sandbox: &Sandbox, port: u16, requests: &str) -> Result { + let output = run_python( + sandbox, + PIPELINE_CLIENT, + &[TEST_HOST, &port.to_string(), requests], + ) + .await?; + let result = json_result(&output)?; + let statuses: Vec<&str> = result["statuses"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + testkit::ensure!( + statuses.len() == 2 + && statuses[0].starts_with("HTTP/1.1 200") + && statuses.contains(&"HTTP/1.1 403 Forbidden"), + "expected 200 then 403; got {statuses:?}" + ); + Ok(()) +} + +/// Run [`STATUS_CLIENT`] and fail unless both its requests got a status +/// `expected` accepts. A failed connection counts as status 0. +async fn expect_statuses(sandbox: &Sandbox, port: &str, expected: impl Fn(i64) -> bool) -> Result { + let output = run_python(sandbox, STATUS_CLIENT, &[TEST_HOST, port]).await?; + let result = json_result(&output)?; + for request in ["first", "second"] { + let status = result[request].as_i64().unwrap_or(0); + testkit::ensure!( + expected(status), + "unexpected {request} request outcome: {result}" + ); + } + Ok(()) +} + +/// The sandbox's policy revisions, from `policy list --output json`. +async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { + cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) + .await? + .success()? + .json() +} + +/// The contents of `path` in the sandbox, once it exists. A timeout reports +/// the keep-alive client's log. +async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { + Ok(sandbox + .poll_exec( + &format!("{path} to appear"), + Duration::from_secs(60), + [ + "sh", + "-c", + &format!("cat {path} || {{ cat {RELOAD_LOG}; exit 1; }}"), + ], + |output| output.exit_code == 0, + ) + .await? + .stdout) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/tests/suites/conformance/src/scripts/echo_client.py b/tests/suites/conformance/src/scripts/echo_client.py new file mode 100644 index 0000000000..aea3b9da97 --- /dev/null +++ b/tests/suites/conformance/src/scripts/echo_client.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends the bytes hex-encoded in argv[3] to the echo server at +# argv[1]:argv[2] and fails unless it reads the same bytes back. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = bytes.fromhex(sys.argv[3]) + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk +if echoed != PAYLOAD: + raise SystemExit(f"the payload changed in transit: {echoed!r}") +print("ECHO_OK") diff --git a/tests/suites/conformance/src/scripts/pipeline_client.py b/tests/suites/conformance/src/scripts/pipeline_client.py new file mode 100644 index 0000000000..2ed5521bd9 --- /dev/null +++ b/tests/suites/conformance/src/scripts/pipeline_client.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends the pipelined HTTP requests in argv[3] to argv[1]:argv[2] in one +# write, reads until the connection closes, and prints the status lines of +# the responses. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +REQUESTS = sys.argv[3].encode() + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(REQUESTS) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk +statuses = [ + line.decode(errors="replace") + for line in response.split(b"\r\n") + if line.startswith(b"HTTP/1.1 ") +] +print(json.dumps({"statuses": statuses})) diff --git a/tests/suites/conformance/src/scripts/redaction_client.py b/tests/suites/conformance/src/scripts/redaction_client.py new file mode 100644 index 0000000000..6bb7bc9b99 --- /dev/null +++ b/tests/suites/conformance/src/scripts/redaction_client.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# POSTs a JSON body holding an API key to the body-echo server at +# argv[1]:argv[2] twice, each on its own connection, and prints the bodies +# the server echoed. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +SECRET = "sk-1234567890abcdef" + + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + if status != 200: + raise RuntimeError(f"request failed with HTTP {status}: {body!r}") + return json.loads(body[:length]) + + +def request_once(): + body = json.dumps({"api_key": SECRET}, separators=(",", ":")).encode() + request = ( + "POST /middleware HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request) + return read_response(sock) + + +print(json.dumps({"first": request_once(), "second": request_once()}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/reload_keep_alive_client.py b/tests/suites/conformance/src/scripts/reload_keep_alive_client.py new file mode 100644 index 0000000000..906cb95019 --- /dev/null +++ b/tests/suites/conformance/src/scripts/reload_keep_alive_client.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends a keep-alive GET to argv[1]:argv[2] and touches READY once it +# succeeds. When GO appears, sends a second request on the same connection +# and writes to RESULT whether it was refused. +import json +import os +import socket +import sys +import time + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +READY = "/tmp/proxy-reload-ready" +GO = "/tmp/proxy-reload-go" +RESULT = "/tmp/proxy-reload-result" + + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + return 0 + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + return 0 + body += chunk + return int(headers.split(None, 2)[1]) + + +target = f"{HOST}:{PORT}" +failed_closed = False +second_status = 0 +try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET /before-reload HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n".encode() + ) + if read_response(sock) != 200: + raise RuntimeError("the first request was denied") + open(READY, "w").close() + deadline = time.monotonic() + 120 + while not os.path.exists(GO) and time.monotonic() < deadline: + time.sleep(0.1) + if not os.path.exists(GO): + raise RuntimeError("timed out waiting for the policy reload") + try: + sock.sendall( + f"GET /after-reload HTTP/1.1\r\nHost: {target}\r\nConnection: close\r\n\r\n".encode() + ) + second_status = read_response(sock) + except OSError: + second_status = 0 + failed_closed = second_status != 200 +finally: + with open(RESULT, "w") as result: + json.dump({"failed_closed": failed_closed, "second_status": second_status}, result, sort_keys=True) diff --git a/tests/suites/conformance/src/scripts/status_client.py b/tests/suites/conformance/src/scripts/status_client.py new file mode 100644 index 0000000000..1d059a9c37 --- /dev/null +++ b/tests/suites/conformance/src/scripts/status_client.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends two GET requests to argv[1]:argv[2], each on its own connection, and +# prints their statuses, or the error a connection failed with. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + + +def read_headers(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + + +def status(response): + parts = response.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + + +def request_status(path): + try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET {path} HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + except OSError as error: + return {"errno": error.errno, "error": repr(error)} + + +print(json.dumps({ + "first": request_status("/first"), + "second": request_status("/second"), +}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/uninspectable_client.py b/tests/suites/conformance/src/scripts/uninspectable_client.py new file mode 100644 index 0000000000..75fa7d65a7 --- /dev/null +++ b/tests/suites/conformance/src/scripts/uninspectable_client.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends bytes that are neither HTTP nor TLS to argv[1]:argv[2] and fails if +# the connection answers with anything but a 403 unsupported_l7_protocol +# denial. A reset or an empty close also counts as blocked. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = bytes([0x00, 0xFF, 0x13, 0x37]) + b"not-http-or-tls" + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(PAYLOAD) + denial = b"" + while True: + try: + chunk = sock.recv(4096) + except ConnectionResetError: + break + if not chunk: + break + denial += chunk +if denial and ( + b"HTTP/1.1 403 Forbidden" not in denial + or b"unsupported_l7_protocol" not in denial +): + raise SystemExit(f"missing the middleware denial: {denial!r}") +print("BLOCKED") From e8f47da1b8f2da0985761d9ab23cea130f833cc4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 17/63] test(conformance): port credential header, body, and WebSocket text rewrite tests Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 156 +++++++++++++++--- .../src/scripts/credential_rewrite_client.py | 46 ++++++ .../src/scripts/websocket_text_client.py | 74 +++++++++ 3 files changed, 250 insertions(+), 26 deletions(-) create mode 100644 tests/suites/conformance/src/scripts/credential_rewrite_client.py create mode 100644 tests/suites/conformance/src/scripts/websocket_text_client.py diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 0b4c4e6f29..f73bc46762 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -4,35 +4,41 @@ //! Credential gating: which endpoints a provider credential may reach, and //! what the supervisor forwards upstream in REST bodies and WebSocket frames. //! -//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream +//! Ported from the e2e tests added in NVIDIA/OpenShell#5264, the credential +//! test in `proxy_egress_pipeline`, and `websocket_conformance`. The upstream //! servers run on the test host ([`host_server_support`]), so the suite //! requires sandboxes to reach the test host as `host.openshell.internal`. use std::time::Duration; -use host_server_support::{HttpRecorder, Request, TEST_HOST, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, TEXT_REPLY, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; +use crate::support::{self, TEST_HOST, python_binaries_yaml}; + +const CREDENTIAL_REWRITE_CLIENT: &str = include_str!("scripts/credential_rewrite_client.py"); +const WEBSOCKET_TEXT_CLIENT: &str = include_str!("scripts/websocket_text_client.py"); + const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -/// POSTs a body containing the provider's token to the host in `argv[1]` on -/// the port in `argv[2]`, directly rather than through a proxy. +/// POSTs a body containing the provider's token to `TEST_HOST` on the port +/// in `argv[1]`, directly rather than through a proxy. const BODY_CLIENT: &str = r#" import os, sys, urllib.request body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() -request = urllib.request.Request(f"http://{sys.argv[1]}:{sys.argv[2]}/token", data=body, headers={"Content-Type": "text/plain"}) +request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() "#; -/// Upgrades to WebSocket at the host in `argv[1]` on the port in `argv[2]`, -/// sends one binary frame, and waits briefly for the connection to be closed. +/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one +/// binary frame, and waits briefly for the connection to be closed. const WEBSOCKET_CLIENT: &str = r#" import base64, os, socket, sys -host, port = sys.argv[1], int(sys.argv[2]) +host, port = "host.openshell.internal", int(sys.argv[1]) with socket.create_connection((host, port), timeout=10) as sock: key = base64.b64encode(os.urandom(16)).decode() sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) @@ -216,6 +222,84 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t wait_for_token(&sandbox, "TOKEN_ABSENT").await } +/// A provider whose profile declares a REST endpoint with body rewriting +/// gets its placeholder replaced with the secret in both the +/// `Authorization` header and the body of every request, with no +/// placeholder left upstream. +#[testkit::test] +async fn profile_rest_endpoint_rewrites_header_and_body(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let sandbox = profile_endpoint_sandbox( + cx, + "rewrite", + &format!( + "{{host: {TEST_HOST}, port: {}, path: /probe, protocol: rest, access: full, \ + enforcement: enforce, request_body_credential_rewrite: true}}", + server.port + ), + ) + .await?; + support::run_python( + &sandbox, + CREDENTIAL_REWRITE_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout("REWRITE_OK\n")?; + + let requests = server.wait_for_requests(2).await?; + let authorization = format!("Bearer {TEST_SECRET}"); + for request in &requests { + testkit::ensure!( + request.header("authorization").as_deref() == Some(authorization.as_str()) + && request.body.contains(TEST_SECRET) + && !request.head.contains(PLACEHOLDER_PREFIX) + && !request.body.contains(PLACEHOLDER_PREFIX), + "expected the secret in the header and body upstream; got {}", + redact(request) + ); + } + Ok(()) +} + +/// A provider whose profile declares a WebSocket endpoint with +/// `websocket_credential_rewrite` gets its placeholder replaced with the +/// secret in a text frame. +#[testkit::test] +async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> testkit::Result { + let server = WebSocketRecorder::start().await?; + let sandbox = profile_endpoint_sandbox( + cx, + "wstext", + &format!( + "{{host: {TEST_HOST}, port: {}, path: /ws, protocol: websocket, \ + enforcement: enforce, access: read-write, websocket_credential_rewrite: true}}", + server.port + ), + ) + .await?; + support::run_python( + &sandbox, + WEBSOCKET_TEXT_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout(&format!("{TEXT_REPLY}\n"))?; + + let texts = server.texts(); + testkit::ensure!( + texts.len() == 1 + && texts[0].contains(TEST_SECRET) + && !texts[0].contains(PLACEHOLDER_PREFIX), + "expected one text frame with the secret upstream; got {:?}", + texts + .iter() + .map(|text| text.replace(TEST_SECRET, "")) + .collect::>() + ); + Ok(()) +} + /// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, /// and return whether its binary frame reached `server`. async fn send_binary_frame( @@ -250,21 +334,43 @@ fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEnd /// `endpoints`, its profile declares the REST and WebSocket ports as its /// endpoints; without, a policy must bind the provider explicitly. async fn install_gating_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { - let name = cx.name("cg"); - let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { - format!( - "endpoints: - - host: {TEST_HOST} - port: {rest} - protocol: rest - access: full - - host: {TEST_HOST} - port: {websocket} - protocol: websocket - access: read-write -" - ) + let endpoints = endpoints.map_or_else(Vec::new, |(rest, websocket)| { + vec![ + format!("{{host: {TEST_HOST}, port: {rest}, protocol: rest, access: full}}"), + format!( + "{{host: {TEST_HOST}, port: {websocket}, protocol: websocket, access: read-write}}" + ), + ] }); + install_provider_with_endpoints(cx, &endpoints).await +} + +/// A long-running sandbox with a provider from +/// [`install_provider_with_endpoints`] whose profile declares `endpoint`, a +/// YAML flow mapping, and a policy that allows the same endpoint. +async fn profile_endpoint_sandbox(cx: &Context, suffix: &str, endpoint: &str) -> Result { + let provider = install_provider_with_endpoints(cx, &[endpoint.to_string()]).await?; + let policy = policy::base_with(&format!( + "network_policies: + credential_gating: + name: credential_gating + endpoints: [{endpoint}] + binaries: {binaries} +", + binaries = python_binaries_yaml(), + ))?; + create_sandbox(cx, suffix, policy, &[&provider]).await +} + +/// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`, whose +/// profile declares `endpoints`, each a YAML flow mapping. +async fn install_provider_with_endpoints(cx: &Context, endpoints: &[String]) -> Result { + let name = cx.name("cg"); + let endpoints = if endpoints.is_empty() { + String::new() + } else { + format!("endpoints: [{}]\n", endpoints.join(", ")) + }; let binaries: String = PYTHON_BINARIES .iter() .map(|path| format!(" - path: {path}\n")) @@ -333,12 +439,11 @@ async fn expect_rejected( } } -/// Run `script` with the image's Python, and [`TEST_HOST`] and `port` as its -/// arguments. +/// Run `script` with the image's Python and `port` as its argument. async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { let port = port.to_string(); sandbox - .exec(["python3", "-c", script, TEST_HOST, &port]) + .exec(["python3", "-c", script, &port]) .await? .success() .context("run the client in the sandbox")?; @@ -390,4 +495,3 @@ async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { .await?; Ok(()) } - diff --git a/tests/suites/conformance/src/scripts/credential_rewrite_client.py b/tests/suites/conformance/src/scripts/credential_rewrite_client.py new file mode 100644 index 0000000000..b3d77a3ac0 --- /dev/null +++ b/tests/suites/conformance/src/scripts/credential_rewrite_client.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# POSTs the provider's token from E2E_GATING_TOKEN in an Authorization header +# and a JSON body to argv[1]:argv[2]/probe twice, each on its own +# connection, and fails unless both get 200. Never prints the token. +import json +import os +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +TOKEN = os.environ["E2E_GATING_TOKEN"] + + +def status(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + parts = data.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + + +def request_once(): + body = json.dumps({"credential": TOKEN}, separators=(",", ":")).encode() + request = ( + "POST /probe HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + f"Authorization: Bearer {TOKEN}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request) + return status(sock) + + +statuses = [request_once(), request_once()] +if statuses != [200, 200]: + raise SystemExit(f"expected two 200 responses; got {statuses}") +print("REWRITE_OK") diff --git a/tests/suites/conformance/src/scripts/websocket_text_client.py b/tests/suites/conformance/src/scripts/websocket_text_client.py new file mode 100644 index 0000000000..a211a83f36 --- /dev/null +++ b/tests/suites/conformance/src/scripts/websocket_text_client.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Upgrades to WebSocket at argv[1]:argv[2], sends the provider's token from +# E2E_GATING_TOKEN in one text frame, and prints the server's text reply. +# Never prints the token. +import base64 +import json +import os +import socket +import struct +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +TOKEN = os.environ["E2E_GATING_TOKEN"] + + +def read_exact(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError("the connection closed mid-frame") + data += chunk + return data + + +def masked_text_frame(text): + data = text.encode() + mask = os.urandom(4) + if len(data) < 126: + header = bytes([0x81, 0x80 | len(data)]) + else: + header = bytes([0x81, 0x80 | 126]) + struct.pack("!H", len(data)) + return header + mask + bytes(byte ^ mask[i % 4] for i, byte in enumerate(data)) + + +def read_text_frame(sock): + _, second = read_exact(sock, 2) + length = second & 0x7F + if length == 126: + length = struct.unpack("!H", read_exact(sock, 2))[0] + elif length == 127: + length = struct.unpack("!Q", read_exact(sock, 8))[0] + mask = read_exact(sock, 4) if second & 0x80 else b"" + payload = read_exact(sock, length) + if mask: + payload = bytes(byte ^ mask[i % 4] for i, byte in enumerate(payload)) + return payload.decode() + + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + key = base64.b64encode(os.urandom(16)).decode() + sock.sendall( + ( + "GET /ws HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n" + ).encode() + ) + response = b"" + while b"\r\n\r\n" not in response: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk + if not response.startswith(b"HTTP/1.1 101"): + raise SystemExit(f"upgrade failed: {response!r}") + sock.sendall(masked_text_frame(json.dumps({"authorization": "Bearer " + TOKEN}))) + print(read_text_frame(sock)) From c5f14c11b40a35785edbde6df461e223fe42ab58 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:59:09 -0800 Subject: [PATCH 18/63] test(conformance): port static provider credential endpoint binding test Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 114 +++++++++++++++++- .../src/scripts/bound_credential_client.sh | 34 ++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/suites/conformance/src/scripts/bound_credential_client.sh diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index f73bc46762..308fb49bd4 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -11,7 +11,8 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, TEXT_REPLY, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, Response, TEXT_REPLY, WebSocketRecorder}; +use serde_json::json; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; @@ -20,6 +21,8 @@ use crate::support::{self, TEST_HOST, python_binaries_yaml}; const CREDENTIAL_REWRITE_CLIENT: &str = include_str!("scripts/credential_rewrite_client.py"); const WEBSOCKET_TEXT_CLIENT: &str = include_str!("scripts/websocket_text_client.py"); +const BOUND_CREDENTIAL_CLIENT: &str = include_str!("scripts/bound_credential_client.sh"); +const BASH: &str = "/usr/bin/bash"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; @@ -300,6 +303,84 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te Ok(()) } +/// A provider's credential resolves only at the endpoints its profile +/// declares: another provider's endpoint and an undeclared path on its own +/// endpoint get `403`, and the sandbox logs the mismatch without the secret +/// or its variable name. +#[testkit::test] +async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit::Result { + const SECRET_A: &str = "e2e-bound-secret"; + const SECRET_B: &str = "e2e-provider-b-secret"; + let authorized = |request: &Request| { + let authorized = request.header("authorization").as_deref() + == Some(format!("Bearer {SECRET_A}").as_str()); + Response::json(&json!({ "authorized": authorized })) + }; + let server_a = HttpRecorder::serve(authorized).await?; + let server_b = HttpRecorder::serve(authorized).await?; + let provider_a = + install_bound_provider(cx, "binda", "BOUND_TOKEN_A", SECRET_A, server_a.port).await?; + let provider_b = + install_bound_provider(cx, "bindb", "BOUND_TOKEN_B", SECRET_B, server_b.port).await?; + + let endpoint = |port| { + format!( + "{{host: {TEST_HOST}, port: {port}, path: /**, protocol: rest, access: full, \ + enforcement: enforce}}" + ) + }; + let policy = policy::base_with(&format!( + "network_policies: + binding: + name: binding + endpoints: [{}, {}] + binaries: [{{path: {BASH}}}] +", + endpoint(server_a.port), + endpoint(server_b.port), + ))?; + // The default image, like the e2e test this replaces, for its bash. + let sandbox = cx + .sandbox("bind") + .policy(policy) + .spec(|spec| spec.providers = vec![provider_a, provider_b]) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let output = sandbox + .exec([ + BASH, + "-c", + BOUND_CREDENTIAL_CLIENT, + "bash", + TEST_HOST, + &server_a.port.to_string(), + &server_b.port.to_string(), + ]) + .await? + .success()?; + testkit::ensure!( + output.stdout.trim() + == r#"ALLOWED={"authorized":true} ENDPOINT_DENIED=403 PATH_DENIED=403"#, + "expected the credential only at its bound endpoint and path; got {}", + output.stdout.replace(SECRET_A, "") + ); + + let logs = sandbox + .logs_until("the endpoint mismatch", |logs| { + logs.contains("openshell.provider_credential.endpoint_mismatch") + && logs.contains("credential_endpoint_mismatch") + }) + .await?; + for leaked in [SECRET_A, SECRET_B, "BOUND_TOKEN_A", "BOUND_TOKEN_B"] { + testkit::ensure!( + !logs.contains(leaked), + "the sandbox logs contain a credential value or variable name" + ); + } + Ok(()) +} + /// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, /// and return whether its binary frame reached `server`. async fn send_binary_frame( @@ -362,6 +443,37 @@ async fn profile_endpoint_sandbox(cx: &Context, suffix: &str, endpoint: &str) -> create_sandbox(cx, suffix, policy, &[&provider]).await } +/// Install a provider named `cx.name(suffix)` holding `secret` in `env_var`, +/// whose profile binds it to `/allowed/**` on `TEST_HOST` at `port` for +/// [`BASH`]. +async fn install_bound_provider( + cx: &Context, + suffix: &str, + env_var: &str, + secret: &str, + port: u16, +) -> Result { + let name = cx.name(suffix); + let profile = format!( + "id: {name} +display_name: E2E static endpoint binding +category: other +credentials: + - name: bound_token + env_vars: [{env_var}] + required: true + auth_style: bearer + header_name: authorization +endpoints: + - {{host: {TEST_HOST}, port: {port}, path: /allowed/**, protocol: rest, access: full, enforcement: enforce}} +binaries: [{BASH}] +" + ); + cx.install_provider(&name, &profile, &[(env_var, secret)]) + .await?; + Ok(name) +} + /// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`, whose /// profile declares `endpoints`, each a YAML flow mapping. async fn install_provider_with_endpoints(cx: &Context, endpoints: &[String]) -> Result { diff --git a/tests/suites/conformance/src/scripts/bound_credential_client.sh b/tests/suites/conformance/src/scripts/bound_credential_client.sh new file mode 100644 index 0000000000..8ec43ede27 --- /dev/null +++ b/tests/suites/conformance/src/scripts/bound_credential_client.sh @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends provider A's token from BOUND_TOKEN_A to host $1: on port $2 at a +# path its profile binds, on port $3, which provider B's profile binds, and on +# port $2 at a path outside the binding. Prints the first response body and +# the other two statuses. Never prints the token. +host="$1" port_a="$2" port_b="$3" + +http_request() { + local port="$1" path="$2" status_line line + HTTP_STATUS= HTTP_BODY= + if ! exec 3<>"/dev/tcp/$host/$port"; then + HTTP_STATUS=connect-denied + return 1 + fi + printf 'GET %s HTTP/1.1\r\nHost: %s:%s\r\nAuthorization: Bearer %s\r\nConnection: close\r\n\r\n' \ + "$path" "$host" "$port" "$BOUND_TOKEN_A" >&3 + IFS= read -r status_line <&3 || return 1 + status_line="${status_line%$'\r'}" + HTTP_STATUS="${status_line#* }" + HTTP_STATUS="${HTTP_STATUS%% *}" + while IFS= read -r line <&3; do + line="${line%$'\r'}" + [[ -z "$line" ]] && break + done + while IFS= read -r line <&3 || [[ -n "$line" ]]; do HTTP_BODY+="$line"; done + exec 3>&- 3<&- +} + +http_request "$port_a" /allowed/check; allowed="$HTTP_BODY" +http_request "$port_b" /allowed/check || true; endpoint_denied="$HTTP_STATUS" +http_request "$port_a" /other/check; path_denied="$HTTP_STATUS" +printf 'ALLOWED=%s ENDPOINT_DENIED=%s PATH_DENIED=%s\n' "$allowed" "$endpoint_denied" "$path_denied" From d6069f86ff7f9e9dafc9fa7969411e209376144f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:03:42 -0800 Subject: [PATCH 19/63] test(conformance): port live policy update tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + .../suites/conformance/src/network_egress.rs | 11 +- tests/suites/conformance/src/policy_update.rs | 364 ++++++++++++++++++ tests/suites/conformance/src/support.rs | 10 +- 4 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 tests/suites/conformance/src/policy_update.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 7c40b58bd2..e9cdf78b84 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -23,6 +23,7 @@ mod network_egress; mod network_l7; mod persistent_workspace; mod policy_advisor; +mod policy_update; mod process; mod settings; mod smoke; diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 8a63120681..f29d8fa63a 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -19,7 +19,8 @@ use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; use crate::support::{ - TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, + TEST_HOST, expect_values, json_result, policy_revisions, python_binaries_yaml, python_sandbox, + run_python, }; const STATUS_CLIENT: &str = include_str!("scripts/status_client.py"); @@ -408,14 +409,6 @@ async fn expect_statuses(sandbox: &Sandbox, port: &str, expected: impl Fn(i64) - Ok(()) } -/// The sandbox's policy revisions, from `policy list --output json`. -async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { - cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) - .await? - .success()? - .json() -} - /// The contents of `path` in the sandbox, once it exists. A timeout reports /// the keep-alive client's log. async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { diff --git a/tests/suites/conformance/src/policy_update.rs b/tests/suites/conformance/src/policy_update.rs new file mode 100644 index 0000000000..a55efa1d58 --- /dev/null +++ b/tests/suites/conformance/src/policy_update.rs @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Live policy updates through `policy set` and `policy update`: revisions +//! and versions, idempotent updates, scoped rule changes, and the revision a +//! supervisor writes when it fills in a sparse initial policy. +//! +//! Ported from the `live_policy_update` e2e tests. Revisions are read from +//! `policy get` and `policy list` with `--output json`. + +use std::time::Duration; + +use serde_json::Value; +use testkit::policy; +use testkit::sdk::raw::proto; +use testkit::{Context, Result, Sandbox}; + +use crate::support::policy_revisions; + +/// A network-only policy the supervisor fills in with the filesystem paths it +/// needs at startup, which makes a second revision. +const SPARSE_POLICY: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../examples/policy-advisor/sandbox-policy.yaml" +)); + +/// How long a revision may take to be loaded. +const LOAD_TIMEOUT: Duration = Duration::from_mins(2); + +/// Each new policy makes a newer version with a different hash, and setting +/// the same policy again makes no new version. Starts from a policy without +/// network rules, adds one rule, then a second. Every revision is listed, and +/// none is left pending. +#[testkit::test] +async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("polset") + .policy(policy::base()) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let initial = settled(cx, &sandbox).await?; + + let mut one_rule = policy::base(); + policy::allow(&mut one_rule, "rule_0", "api.anthropic.com", 443, &["/**"]); + let mut two_rules = one_rule.clone(); + policy::allow(&mut two_rules, "rule_1", "example.com", 443, &["/**"]); + + let mut previous = initial.clone(); + let mut versions = vec![version(&initial)?]; + for (step, next) in [("one rule", &one_rule), ("two rules", &two_rules)] { + policy::set(&sandbox, next).await?; + let updated = settled(cx, &sandbox).await?; + testkit::ensure!( + version(&updated)? > version(&previous)?, + "setting {step} didn't make a newer version: before {previous}, after {updated}" + ); + testkit::ensure!( + updated["hash"] != previous["hash"], + "setting {step} didn't change the hash: {}", + updated["hash"] + ); + + policy::set(&sandbox, next).await?; + let repeated = settled(cx, &sandbox).await?; + for field in ["version", "hash"] { + testkit::ensure!( + repeated[field] == updated[field], + "setting {step} again changed the {field}: {} became {}", + updated[field], + repeated[field] + ); + } + versions.push(version(&updated)?); + previous = updated; + } + + let revisions = policy_revisions(cx, &sandbox).await?; + let listed: Vec = revisions["revisions"] + .as_array() + .into_iter() + .flatten() + .filter_map(|revision| revision["version"].as_u64()) + .collect(); + let missing: Vec<_> = versions.iter().filter(|v| !listed.contains(v)).collect(); + testkit::ensure!( + missing.is_empty(), + "policy list is missing versions {missing:?}: {revisions}" + ); + Ok(()) +} + +/// `policy update` with an explicit rule and binaries changes only the +/// endpoint they select, even when another rule has the same host and ports. +/// An update whose ports or binaries match the rule only in part is rejected +/// without a new revision. +#[testkit::test] +async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("polscope") + .policy(scoped_policy()?) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let before = settled(cx, &sandbox).await?; + let rules_before = &before["policy"]["network_policies"]; + testkit::ensure!( + rules_before.is_object(), + "policy get has no network policies: {before}" + ); + + // Each partial update matches one of ports and binaries fully, so a check + // that needs both to mismatch can't pass. + for (target, binaries) in [ + ( + "api.example.com:443:POST:/admin", + &["/usr/bin/curl", "/usr/bin/python3"][..], + ), + ( + "api.example.com:443,8443:POST:/admin", + &["/usr/bin/curl"][..], + ), + ] { + let mut args = vec![ + "policy", + "update", + sandbox.name(), + "--rule-name", + "internal_api", + "--add-allow", + target, + ]; + for binary in binaries { + args.extend(["--binary", binary]); + } + cx.cli(args).await?.failure()?; + let unchanged = policy_get(cx, &sandbox).await?; + for field in ["version", "hash", "policy"] { + testkit::ensure!( + unchanged[field] == before[field], + "rejected update {target} with {binaries:?} changed the {field}" + ); + } + } + + update( + cx, + &sandbox, + "--add-allow", + "api.example.com:443,8443:POST:/admin", + ) + .await?; + let allowed = policy_get(cx, &sandbox).await?; + testkit::ensure!( + version(&allowed)? > version(&before)? && allowed["hash"] != before["hash"], + "the allow didn't make a new revision: before {before}, after {allowed}" + ); + let rules = &allowed["policy"]["network_policies"]; + testkit::ensure!( + rules["sibling"] == rules_before["sibling"], + "the allow changed the sibling rule: {}", + rules["sibling"] + ); + testkit::ensure!( + rules["internal_api"]["endpoints"][1] == rules_before["internal_api"]["endpoints"][1], + "the allow changed the other endpoint: {}", + rules["internal_api"]["endpoints"][1] + ); + testkit::ensure!( + has_rule(&rules["internal_api"]["endpoints"][0]["rules"], |rule| { + rule["allow"]["method"] == "POST" && rule["allow"]["path"] == "/admin" + }), + "the endpoint has no POST /admin allow rule: {}", + rules["internal_api"]["endpoints"][0] + ); + + update( + cx, + &sandbox, + "--add-deny", + "api.example.com:443,8443:POST:/admin/private", + ) + .await?; + let denied = policy_get(cx, &sandbox).await?; + testkit::ensure!( + version(&denied)? > version(&allowed)?, + "the deny didn't make a new revision: before {allowed}, after {denied}" + ); + let rules = &denied["policy"]["network_policies"]; + testkit::ensure!( + rules["sibling"] == rules_before["sibling"], + "the deny changed the sibling rule: {}", + rules["sibling"] + ); + testkit::ensure!( + has_rule( + &rules["internal_api"]["endpoints"][0]["deny_rules"], + |rule| { rule["method"] == "POST" && rule["path"] == "/admin/private" } + ), + "the endpoint has no POST /admin/private deny rule: {}", + rules["internal_api"]["endpoints"][0] + ); + Ok(()) +} + +/// A sandbox created with a sparse policy gets a second revision with the +/// paths the supervisor added, and the supervisor reports it loaded: the +/// first is superseded and neither stays pending. Regression for #2159. +#[testkit::test] +async fn sparse_initial_policy_revision_is_loaded(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .cli_sandbox("sparse") + .policy_yaml(SPARSE_POLICY) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + testkit::poll_until( + "the filled-in revision 2 to be loaded", + LOAD_TIMEOUT, + Duration::from_secs(2), + async || { + let active = policy_get(cx, &sandbox).await?; + let revisions = policy_revisions(cx, &sandbox).await?; + let statuses = statuses(&revisions); + let expected = [(1_u64, "superseded".to_string()), (2, "loaded".to_string())]; + Ok( + if active["version"] == 2 && statuses.as_slice() == expected { + testkit::Poll::Ready(()) + } else { + testkit::Poll::Pending(format!( + "active version {}, revisions {statuses:?}", + active["version"] + )) + }, + ) + }, + ) + .await +} + +/// [`policy::base`] with two rules that share a host and ports: `internal_api` +/// with a second endpoint, and `sibling` for other binaries. +fn scoped_policy() -> Result { + policy::base_with( + "network_policies: + internal_api: + name: internal_api + binaries: [{path: /usr/bin/curl}, {path: /usr/bin/python3}] + endpoints: + - {host: api.example.com, ports: [443, 8443], protocol: rest, access: read-only} + - {host: other.example.com, port: 443, protocol: rest, access: read-only} + sibling: + name: sibling + binaries: [{path: /usr/bin/wget}] + endpoints: + - {host: api.example.com, ports: [443, 8443], protocol: rest, access: read-only} +", + ) +} + +/// Add `rule` to the `internal_api` rule for curl and Python with +/// `policy update --wait`. +async fn update(cx: &Context, sandbox: &Sandbox, flag: &str, rule: &str) -> Result { + cx.cli([ + "policy", + "update", + sandbox.name(), + "--rule-name", + "internal_api", + "--binary", + "/usr/bin/curl", + "--binary", + "/usr/bin/python3", + flag, + rule, + "--wait", + ]) + .await? + .success()?; + Ok(()) +} + +/// The sandbox's active policy, from `policy get --full --output json`. +async fn policy_get(cx: &Context, sandbox: &Sandbox) -> Result { + let active: Value = cx + .cli([ + "policy", + "get", + sandbox.name(), + "--full", + "--output", + "json", + ]) + .await? + .success()? + .json()?; + version(&active)?; + testkit::ensure!( + active["hash"].as_str().is_some_and(|hash| !hash.is_empty()), + "policy get has no hash: {active}" + ); + Ok(active) +} + +/// The active policy once its revision is loaded and no revision is pending. +/// Fails if a revision failed to load. +async fn settled(cx: &Context, sandbox: &Sandbox) -> Result { + testkit::poll_until( + &format!("the policy of sandbox {} to be loaded", sandbox.name()), + LOAD_TIMEOUT, + Duration::from_secs(1), + async || { + let active = policy_get(cx, sandbox).await?; + let revisions = policy_revisions(cx, sandbox).await?; + let statuses = statuses(&revisions); + testkit::ensure!( + statuses.iter().all(|(_, status)| status != "failed"), + "a policy revision failed to load: {revisions}" + ); + let active_version = version(&active)?; + let loaded = statuses + .iter() + .any(|(version, status)| *version == active_version && status == "loaded"); + let pending = statuses.iter().any(|(_, status)| status == "pending"); + Ok(if loaded && !pending { + testkit::Poll::Ready(active) + } else { + testkit::Poll::Pending(format!( + "active version {active_version}, revisions {statuses:?}" + )) + }) + }, + ) + .await +} + +/// Each revision's version and status, oldest first. +fn statuses(revisions: &Value) -> Vec<(u64, String)> { + let mut statuses: Vec<_> = revisions["revisions"] + .as_array() + .into_iter() + .flatten() + .map(|revision| { + ( + revision["version"].as_u64().unwrap_or_default(), + revision["status"].as_str().unwrap_or_default().to_string(), + ) + }) + .collect(); + statuses.sort(); + statuses +} + +fn version(policy: &Value) -> Result { + policy["version"] + .as_u64() + .ok_or_else(|| testkit::Error::msg(format!("policy get has no version: {policy}"))) +} + +fn has_rule(rules: &Value, matches: impl Fn(&Value) -> bool) -> bool { + rules + .as_array() + .is_some_and(|rules| rules.iter().any(matches)) +} diff --git a/tests/suites/conformance/src/support.rs b/tests/suites/conformance/src/support.rs index dd512371df..5b80d1e5a6 100644 --- a/tests/suites/conformance/src/support.rs +++ b/tests/suites/conformance/src/support.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Helpers the network suites share. Not a suite. +//! Helpers the network and policy suites share. Not a suite. //! //! Their clients are Python scripts in `src/scripts/` that take the server's //! host and port as arguments, run in a long-running sandbox from @@ -69,6 +69,14 @@ pub fn expect_values>(result: &Value, expected: &[(K, Value)]) -> Ok(()) } +/// The sandbox's policy revisions, from `policy list --output json`. +pub async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { + cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) + .await? + .success()? + .json() +} + /// `binaries:` for policy YAML that lets the image's Python connect. pub fn python_binaries_yaml() -> String { let paths: Vec = PYTHON_BINARIES From b2cdd6de675779f43233ebc70793f4c77385d41f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:35:32 -0800 Subject: [PATCH 20/63] test(support): add a loopback TCP exchange client to host-server Signed-off-by: Matthew Grossman --- tests/support/host-server/src/lib.rs | 5 ++-- tests/support/host-server/src/tcp.rs | 38 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 5df228f90a..e9781bf502 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -16,7 +16,8 @@ //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the //! client goes idle. -//! - [`free_port`] finds a port nothing listens on. +//! - [`free_port`] finds a port nothing listens on, and [`exchange`] sends +//! bytes to a loopback port and reads the reply. mod http; mod tcp; @@ -28,7 +29,7 @@ use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; pub use http::{HttpRecorder, Request, Response}; -pub use tcp::TcpServer; +pub use tcp::{TcpServer, exchange}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; /// The name a sandbox reaches these servers by. diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs index 42c4eaa9b0..1b6d566895 100644 --- a/tests/support/host-server/src/tcp.rs +++ b/tests/support/host-server/src/tcp.rs @@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use testkit::{Poll, Result}; +use testkit::{Poll, Result, ResultExt as _}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::TcpStream; use tokio::task::JoinHandle; @@ -100,6 +100,35 @@ impl TcpServer { } } +/// Connect to `port` on the test host's loopback, send `request`, and return +/// what arrives until the other end closes. For a port the test reaches from +/// the host, such as one `openshell forward` listens on. +pub async fn exchange(port: u16, request: &[u8]) -> Result> { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let mut stream = TcpStream::connect(("127.0.0.1", port)) + .await + .context(format!("connect to 127.0.0.1:{port}"))?; + stream.set_nodelay(true).context("enable TCP_NODELAY")?; + stream + .write_all(request) + .await + .context(format!("send to 127.0.0.1:{port}"))?; + let mut reply = Vec::new(); + tokio::time::timeout(READ_TIMEOUT, stream.read_to_end(&mut reply)) + .await + .map_err(|_| { + testkit::Error::msg(format!( + "127.0.0.1:{port} didn't close within {READ_TIMEOUT:?}; read {} byte(s)", + reply.len() + )) + })? + .context(format!("read from 127.0.0.1:{port}"))?; + Ok(reply) +} + impl Drop for TcpServer { fn drop(&mut self) { self.task.abort(); @@ -171,4 +200,11 @@ mod tests { assert_eq!(reply, b"reply"); assert_eq!(server.received(), b"first second"); } + + #[tokio::test] + async fn exchange_returns_the_reply_until_close() { + let server = TcpServer::reply_when_idle(b"reply".to_vec()).await.unwrap(); + assert_eq!(exchange(server.port, b"hello").await.unwrap(), b"reply"); + assert_eq!(server.received(), b"hello"); + } } From 326a80670a555ea2f34c3d94b38ffde4d1cba9c4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:37:06 -0800 Subject: [PATCH 21/63] test(conformance): port the port forward test Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/port_forward.rs | 90 +++++++++++++++++++ .../conformance/src/scripts/echo_server.py | 19 ++++ 3 files changed, 110 insertions(+) create mode 100644 tests/suites/conformance/src/port_forward.rs create mode 100644 tests/suites/conformance/src/scripts/echo_server.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index e9cdf78b84..8bd5846c89 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -24,6 +24,7 @@ mod network_l7; mod persistent_workspace; mod policy_advisor; mod policy_update; +mod port_forward; mod process; mod settings; mod smoke; diff --git a/tests/suites/conformance/src/port_forward.rs b/tests/suites/conformance/src/port_forward.rs new file mode 100644 index 0000000000..999a7988a2 --- /dev/null +++ b/tests/suites/conformance/src/port_forward.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Port forwarding with `openshell forward`. +//! +//! Ported from the `port_forward` e2e test. + +use std::time::Duration; + +use testkit::{Context, policy}; + +const ECHO_SERVER: &str = include_str!("scripts/echo_server.py"); + +/// How long `forward start` and the first echo through it may take. +const FORWARD_TIMEOUT: Duration = Duration::from_secs(60); + +/// `forward start -d` forwards a local port to a server on the sandbox's +/// loopback, and `forward stop` ends it. +#[testkit::test] +async fn forward_reaches_sandbox_loopback_server(cx: &mut Context) -> testkit::Result { + let local_port = host_server_support::free_port().await?; + let port = local_port.to_string(); + let sandbox = cx + .sandbox("fwd") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["python3", "-c", ECHO_SERVER, &port]) + .create_ready() + .await?; + let name = sandbox.name().to_string(); + + // The forward's ssh keeps the CLI's stdout and stderr open, so wait for + // the CLI to exit rather than for its output to end. + let mut start = cx.cli_spawn(["forward", "start", "-d", &port, &name])?; + let code = start.wait(FORWARD_TIMEOUT).await?; + if code != 0 { + let output = start.output(FORWARD_TIMEOUT).await?; + return Err(testkit::Error::msg(format!( + "`{}` exited {code}:\n{}", + output.command, + output.report_text() + ))); + } + // Stops the forward if the test fails before it does. Its outcome is + // ignored, since the forward is already gone when the test passes. + let cli = cx.cli_handle(); + let (stop_port, stop_name) = (port.clone(), name.clone()); + cx.on_cleanup(format!("stop forward {port}"), async move || { + cli.run(["forward", "stop", &stop_port, &stop_name]).await?; + Ok(()) + }); + + testkit::poll_until( + &format!("an echo through forwarded port {port}"), + FORWARD_TIMEOUT, + Duration::from_secs(1), + async || { + Ok( + match host_server_support::exchange(local_port, b"hello-forward\n").await { + Ok(reply) if reply == b"echo:hello-forward\n" => testkit::Poll::Ready(()), + Ok(reply) => testkit::Poll::Pending(format!( + "reply {:?}", + String::from_utf8_lossy(&reply) + )), + Err(error) => testkit::Poll::Pending(error.to_string()), + }, + ) + }, + ) + .await?; + + cx.cli(["forward", "stop", &port, &name]).await?.success()?; + testkit::poll_until( + &format!("forwarded port {port} to close"), + FORWARD_TIMEOUT, + Duration::from_secs(1), + async || { + Ok( + match host_server_support::exchange(local_port, b"after-stop\n").await { + Err(_) => testkit::Poll::Ready(()), + Ok(reply) => testkit::Poll::Pending(format!( + "still forwarding; reply {:?}", + String::from_utf8_lossy(&reply) + )), + }, + ) + }, + ) + .await +} diff --git a/tests/suites/conformance/src/scripts/echo_server.py b/tests/suites/conformance/src/scripts/echo_server.py new file mode 100644 index 0000000000..9933a17333 --- /dev/null +++ b/tests/suites/conformance/src/scripts/echo_server.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A TCP server on 127.0.0.1:argv[1] that answers each connection's first +# read with b"echo:" and the bytes read, then closes it. + +import socket +import sys + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(("127.0.0.1", int(sys.argv[1]))) +server.listen(8) +while True: + conn, _ = server.accept() + with conn: + data = conn.recv(4096) + if data: + conn.sendall(b"echo:" + data) From a2164e8a2e150ceb56a168a8816bfb3ab4a1d521 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:39:04 -0800 Subject: [PATCH 22/63] test(features): port the IP literal egress test to routable-host-ip Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 4 + tests/suites/features/Cargo.lock | 9 ++ tests/suites/features/Cargo.toml | 2 +- .../features/routable-host-ip/Cargo.toml | 17 ++++ .../tests/routable_host_ip.rs | 96 +++++++++++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/suites/features/routable-host-ip/Cargo.toml create mode 100644 tests/suites/features/routable-host-ip/tests/routable_host_ip.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index c509b907b3..4936fd7952 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -104,6 +104,10 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 0e68a6149b..99f2256d96 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -3115,6 +3115,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "routable-host-ip" +version = "0.0.0" +dependencies = [ + "host-server-support", + "openshell-testkit", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.27" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index e12a0a1a02..7c7fe40ae7 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "routable-host-ip"] diff --git a/tests/suites/features/routable-host-ip/Cargo.toml b/tests/suites/features/routable-host-ip/Cargo.toml new file mode 100644 index 0000000000..0645be32e8 --- /dev/null +++ b/tests/suites/features/routable-host-ip/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "routable-host-ip" +description = "Features suite: sandboxes reach the test host by an IP literal policy endpoint" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +host-server-support = { path = "../../../support/host-server" } +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs new file mode 100644 index 0000000000..613fe4d0a2 --- /dev/null +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A policy endpoint whose host is an IP literal reaches that address +//! transparently, whether `allowed_ips` names it or not. +//! +//! The servers run on the test host, and the sandbox reaches them by the +//! address `host.openshell.internal` resolves to. Not conformance, because +//! that address must be one the supervisor lets an IP literal reach: rootless +//! Podman's is link-local, which is always blocked. +//! +//! Ported from the `proxy_egress_pipeline` e2e test of IP literals, which +//! used two support containers on the e2e network instead. + +use host_server_support::HttpRecorder; +use serde_json::Value; +use testkit::policy; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; + +/// Sends two GET requests to `argv[1]:argv[2]` and prints their statuses as +/// one JSON line. +const STATUS_CLIENT: &str = include_str!("../../../conformance/src/scripts/status_client.py"); + +const RESOLVE: &str = "import socket; print(socket.gethostbyname('host.openshell.internal'))"; + +/// Requests reach a server by an IP literal endpoint with `allowed_ips` for +/// that address, and another by an IP literal endpoint without it. +#[testkit::test] +async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Result { + let explicit = HttpRecorder::start("explicit").await?; + let implicit = HttpRecorder::start("implicit").await?; + let sandbox = cx + .sandbox("hostip") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let ip = sandbox + .exec(["python3", "-c", RESOLVE]) + .await? + .success() + .context("resolve host.openshell.internal in the sandbox")? + .stdout + .trim() + .to_string(); + + let mut ip_policy = policy::base(); + policy::allow( + &mut ip_policy, + "explicit", + &ip, + explicit.port, + PYTHON_BINARIES, + ) + .allowed_ips = vec![format!("{ip}/32")]; + policy::allow( + &mut ip_policy, + "implicit", + &ip, + implicit.port, + PYTHON_BINARIES, + ); + policy::set(&sandbox, &ip_policy).await?; + + for (mode, server) in [("explicit allowed_ips", &explicit), ("implicit", &implicit)] { + let output = sandbox + .exec([ + "python3", + "-c", + STATUS_CLIENT, + &ip, + &server.port.to_string(), + ]) + .await? + .success() + .context(format!("run the {mode} client"))?; + let result: Value = output + .stdout + .lines() + .rev() + .find_map(|line| serde_json::from_str(line.trim()).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!( + "no JSON result from the {mode} client:\n{}\n{}", + output.stdout, output.stderr + )) + })?; + testkit::ensure!( + result["first"] == 200 && result["second"] == 200, + "{mode} requests to {ip}:{} failed: {result}", + server.port + ); + } + Ok(()) +} From 98442b2f9fe23a4d6af8d341154f922665ffcd18 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:44:46 -0800 Subject: [PATCH 23/63] test(e2e): remove legacy network, L7, policy update, and port forward tests These tests now run in the conformance suite and the routable-host-ip feature package. Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 25 - e2e/rust/e2e-podman.sh | 6 - e2e/rust/src/harness/sandbox.rs | 20 - e2e/rust/tests/bypass_detection.rs | 79 - e2e/rust/tests/forward_proxy_graphql_l7.rs | 415 ----- e2e/rust/tests/forward_proxy_jsonrpc_l7.rs | 519 ------ e2e/rust/tests/forward_proxy_l7_bypass.rs | 239 --- e2e/rust/tests/host_gateway_alias.rs | 341 +--- e2e/rust/tests/listener_lifetime.rs | 154 -- e2e/rust/tests/live_policy_update.rs | 704 -------- e2e/rust/tests/mcp_sessionless.rs | 381 ----- e2e/rust/tests/no_proxy.rs | 59 - e2e/rust/tests/port_forward.rs | 140 -- e2e/rust/tests/proxy_egress_pipeline.rs | 1683 -------------------- e2e/rust/tests/websocket_conformance.rs | 517 ------ tasks/test.toml | 9 +- tests/tmachine/artifacts.nix | 4 - 17 files changed, 2 insertions(+), 5293 deletions(-) delete mode 100644 e2e/rust/tests/bypass_detection.rs delete mode 100644 e2e/rust/tests/forward_proxy_graphql_l7.rs delete mode 100644 e2e/rust/tests/forward_proxy_jsonrpc_l7.rs delete mode 100644 e2e/rust/tests/forward_proxy_l7_bypass.rs delete mode 100644 e2e/rust/tests/listener_lifetime.rs delete mode 100644 e2e/rust/tests/live_policy_update.rs delete mode 100644 e2e/rust/tests/mcp_sessionless.rs delete mode 100644 e2e/rust/tests/no_proxy.rs delete mode 100644 e2e/rust/tests/port_forward.rs delete mode 100644 e2e/rust/tests/proxy_egress_pipeline.rs delete mode 100644 e2e/rust/tests/websocket_conformance.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 2efe8ed842..89d52187d5 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -198,11 +198,6 @@ name = "credential_drivers" path = "tests/credential_drivers.rs" required-features = ["e2e-kubernetes-credential-drivers"] -[[test]] -name = "websocket_conformance" -path = "tests/websocket_conformance.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "user_namespaces" path = "tests/user_namespaces.rs" @@ -213,31 +208,11 @@ name = "host_gateway_alias" path = "tests/host_gateway_alias.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "forward_proxy_l7_bypass" -path = "tests/forward_proxy_l7_bypass.rs" -required-features = ["e2e-host-gateway"] - -[[test]] -name = "forward_proxy_graphql_l7" -path = "tests/forward_proxy_graphql_l7.rs" -required-features = ["e2e-host-gateway"] - -[[test]] -name = "forward_proxy_jsonrpc_l7" -path = "tests/forward_proxy_jsonrpc_l7.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "supervisor_middleware_http_hooks" path = "tests/supervisor_middleware_http_hooks.rs" required-features = ["e2e-docker"] -[[test]] -name = "proxy_egress_pipeline" -path = "tests/proxy_egress_pipeline.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "workspace_namespace_managed" path = "tests/workspace_namespace_managed.rs" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 3ba8cb2b2f..bd34cf2d77 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -22,17 +22,11 @@ source "${ROOT}/e2e/support/conformance.sh" # unfiltered e2e:podman task remains available while the omitted targets are # stabilized and can be added here. PODMAN_CI_TESTS=( - bypass_detection ephemeral_cleanup driver_config_volume edge_tunnel_e2e - forward_proxy_graphql_l7 - forward_proxy_jsonrpc_l7 - forward_proxy_l7_bypass host_gateway_alias - live_policy_update local_driver_token_restart - no_proxy ocsf_jsonl_conformance otlp_relay podman_corporate_proxy diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index 9293ce2fac..9d14092cb2 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -566,26 +566,6 @@ impl SandboxGuard { Ok(combined) } - /// Spawn `openshell forward start` as a background process. - /// - /// Returns the child process handle. The caller is responsible for killing - /// it (or it will be killed on drop since `kill_on_drop(true)` is set). - /// - /// # Errors - /// - /// Returns an error if the process cannot be spawned. - pub fn spawn_forward(&self, port: u16) -> Result { - let mut cmd = openshell_cmd(); - cmd.arg("forward") - .arg("start") - .arg(port.to_string()) - .arg(&self.name); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - cmd.spawn() - .map_err(|e| format!("failed to spawn port forward: {e}")) - } - /// Delete the sandbox explicitly. /// /// Also kills the background child process if one exists. This is called diff --git a/e2e/rust/tests/bypass_detection.rs b/e2e/rust/tests/bypass_detection.rs deleted file mode 100644 index 569e3a60d6..0000000000 --- a/e2e/rust/tests/bypass_detection.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Verify that direct TCP bypass attempts fail promptly at the sandbox -//! syscall boundary instead of reaching the runtime's external network. -//! -//! This test is implementation-agnostic — it validates the observable -//! behavior rather than a particular packet-filter implementation. - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// Python script that attempts a raw TCP connect bypassing the proxy. -/// -/// `socket.connect()` does not honor proxy environment variables. The script -/// reports the outcome and wall-clock time so the test can assert that the -/// sandbox's seccomp mediation blocks it before the outer fence is needed. -/// -/// Target 198.51.100.1 is RFC 5737 TEST-NET-2 — documentation-only address -/// space that will never route. -fn bypass_attempt_script() -> &'static str { - r#" -import json, socket, time - -start = time.monotonic() -result = "unknown" -try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(10) - s.connect(("198.51.100.1", 80)) - result = "connected" - s.close() -except ConnectionRefusedError: - result = "refused" -except PermissionError: - result = "denied" -except socket.timeout: - result = "timeout" -except OSError as e: - result = f"error:{e}" - -elapsed_ms = int((time.monotonic() - start) * 1000) -print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=True) -"# -} - -/// A direct TCP connection bypassing supervision should be denied without -/// waiting for the socket's network timeout. -#[tokio::test] -async fn bypass_attempt_is_rejected_fast() { - let guard = SandboxGuard::create(&["--", "python3", "-c", bypass_attempt_script()]) - .await - .expect("sandbox create"); - - let json_line = guard - .create_output - .lines() - .find(|l| l.contains("bypass_result")) - .unwrap_or_else(|| panic!("no bypass_result JSON in output:\n{}", guard.create_output)); - - let parsed: serde_json::Value = serde_json::from_str(json_line.trim()) - .unwrap_or_else(|e| panic!("failed to parse JSON '{json_line}': {e}")); - - let result = parsed["bypass_result"].as_str().unwrap(); - let elapsed_ms = parsed["elapsed_ms"].as_u64().unwrap(); - - assert_eq!( - result, "denied", - "expected seccomp mediation to deny the direct connect, got '{result}' after {elapsed_ms}ms.\n\ - Full output:\n{}", - guard.create_output - ); - - assert!( - elapsed_ms < 8000, - "bypass rejection took {elapsed_ms}ms — expected < 8000ms." - ); -} diff --git a/e2e/rust/tests/forward_proxy_graphql_l7.rs b/e2e/rust/tests/forward_proxy_graphql_l7.rs deleted file mode 100644 index 5c25556cfa..0000000000 --- a/e2e/rust/tests/forward_proxy_graphql_l7.rs +++ /dev/null @@ -1,415 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for GraphQL L7 inspection through transparent interception. -//! -//! The upstream server deliberately does not implement GraphQL. `OpenShell` -//! parses and enforces GraphQL before forwarding, so any HTTP server that -//! accepts POST /graphql is enough to prove allowed requests reach upstream -//! and denied requests are stopped by the sandbox proxy. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const TEST_SERVER_ALIAS: &str = "graphql-l7.openshell.test"; - -async fn start_test_server() -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def read_chunked(self): - while True: - size_line = self.rfile.readline() - if not size_line: - return - size = int(size_line.split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - return - self.rfile.read(size) - self.rfile.read(2) - def do_GET(self): - if self.path == "/" or self.path.startswith("/graphql?"): - self.send_response(200) - else: - self.send_response(418) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def do_POST(self): - if self.headers.get("Transfer-Encoding", "").lower() == "chunked": - self.read_chunked() - else: - _ = self.rfile.read(int(self.headers.get("Content-Length", "0"))) - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{"ok":true}') - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await -} - -fn write_graphql_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_graphql_l7: - name: test_graphql_l7 - endpoints: - - host: {host} - port: {port} - protocol: graphql - enforcement: enforce - persisted_queries: allow_registered - graphql_persisted_queries: - abc123: - operation_type: query - operation_name: Viewer - fields: [viewer] - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - rules: - - allow: - operation_type: query - fields: [viewer] - - allow: - operation_type: mutation - fields: [createIssue] - deny_rules: - - operation_type: mutation - fields: [deleteRepository] - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn graphql_l7_enforces_high_level_and_raw_transparent_paths() { - let server = start_test_server().await.expect("start test server"); - let policy = write_graphql_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import socket -import time -import urllib.error -import urllib.parse -import urllib.request - -HOST = {host:?} -PORT = {port} -DETAILS = {{}} - -QUERY_VIEWER = "query Viewer {{ viewer {{ login }} }}" -QUERY_REPOSITORY = "query Repo {{ repository(owner:\"o\", name:\"r\") {{ id }} }}" -MUTATION_CREATE = "mutation Create {{ createIssue(input:{{repositoryId:\"r\", title:\"t\", body:\"b\"}}) {{ issue {{ id }} }} }}" -MUTATION_DELETE = "mutation Delete {{ deleteRepository(input:{{repositoryId:\"r\"}}) {{ clientMutationId }} }}" - -def forward_status(query): - body = json.dumps({{"query": query}}).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql", - data=body, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_get_status(query): - encoded = urllib.parse.urlencode({{"query": query}}) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?{{encoded}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_duplicate_get_status(): - safe = urllib.parse.quote_plus(QUERY_VIEWER) - unsafe = urllib.parse.quote_plus(MUTATION_DELETE) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?query={{safe}}&query={{unsafe}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_persisted_get_status(hash_value): - extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) - encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?{{encoded}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_chunked_status(query): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - - with socket.create_connection((HOST, PORT), timeout=15) as sock: - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk - sock.sendall(request) - response, body = read_response(sock) - DETAILS["forward_chunked_query_allowed_detail"] = body.decode(errors="replace") - return status_code(response, "forward_chunked_response") - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: malformed HTTP response: {{response!r}}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response = read_until(sock, b"\r\n\r\n") - return status_code(response, f"{{label}}_response") - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{{label}}_attempt_{{attempt + 1}}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") - -def raw_status(query, label): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + body - return raw_http_status(label, request) - -def raw_get_status(query, label): - target = f"{{HOST}}:{{PORT}}" - encoded = urllib.parse.urlencode({{"query": query}}) - - request = ( - f"GET /graphql?{{encoded}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_duplicate_get_status(): - target = f"{{HOST}}:{{PORT}}" - safe = urllib.parse.quote_plus(QUERY_VIEWER) - unsafe = urllib.parse.quote_plus(MUTATION_DELETE) - - request = ( - f"GET /graphql?query={{safe}}&query={{unsafe}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status("raw_duplicate_get_denied", request) - -def raw_persisted_get_status(hash_value, label): - target = f"{{HOST}}:{{PORT}}" - extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) - encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) - - request = ( - f"GET /graphql?{{encoded}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_chunked_status(query): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk - return raw_http_status("raw_chunked_query_allowed", request) - -results = {{ - "forward_query_allowed": forward_status(QUERY_VIEWER), - "forward_get_query_allowed": forward_get_status(QUERY_VIEWER), - "forward_duplicate_get_denied": forward_duplicate_get_status(), - "forward_persisted_get_allowed": forward_persisted_get_status("abc123"), - "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"), - "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER), - "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), - "forward_mutation_allowed": forward_status(MUTATION_CREATE), - "forward_deny_rule_denied": forward_status(MUTATION_DELETE), - "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), - "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), - "raw_duplicate_get_denied": raw_duplicate_get_status(), - "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), - "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), - "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), - "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), - "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), - "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), -}} -results.update(DETAILS) -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - ("forward_query_allowed", 200), - ("forward_get_query_allowed", 200), - ("forward_duplicate_get_denied", 403), - ("forward_persisted_get_allowed", 200), - ("forward_unregistered_persisted_get_denied", 403), - ("forward_chunked_query_allowed", 200), - ("forward_unlisted_field_denied", 403), - ("forward_mutation_allowed", 200), - ("forward_deny_rule_denied", 403), - ("raw_query_allowed", 200), - ("raw_get_query_allowed", 200), - ("raw_duplicate_get_denied", 403), - ("raw_persisted_get_allowed", 200), - ("raw_unregistered_persisted_get_denied", 403), - ("raw_chunked_query_allowed", 200), - ("raw_unlisted_field_denied", 403), - ("raw_mutation_allowed", 200), - ("raw_deny_rule_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs deleted file mode 100644 index a5d7c4849d..0000000000 --- a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs +++ /dev/null @@ -1,519 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for JSON-RPC L7 inspection through transparent interception. -//! -//! The upstream server deliberately does not implement JSON-RPC. `OpenShell` -//! parses and enforces JSON-RPC before forwarding, so any HTTP server that -//! accepts POST /rpc is enough to prove allowed requests reach upstream -//! and denied requests are stopped by the sandbox proxy. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const RULES_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-rules.openshell.test"; -const AUDIT_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-audit.openshell.test"; - -async fn start_test_server(alias: &str) -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def read_body(self): - if self.headers.get("Transfer-Encoding", "").lower() == "chunked": - data = b"" - while True: - size_line = self.rfile.readline() - if not size_line: - break - size = int(size_line.split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - break - data += self.rfile.read(size) - self.rfile.read(2) - return data - return self.rfile.read(int(self.headers.get("Content-Length", "0"))) - - def do_GET(self): - self.send_response(200) - self.end_headers() - - def do_POST(self): - self.read_body() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{"jsonrpc":"2.0","id":1,"result":{}}') - - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(alias, script).await -} - -fn write_jsonrpc_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_jsonrpc_l7: - name: test_jsonrpc_l7 - endpoints: - - host: {host} - port: {port} - protocol: rest - enforcement: enforce - access: read-write - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - - host: {host} - port: {port} - path: /rpc - protocol: json-rpc - enforcement: enforce - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - json_rpc: - max_body_bytes: 65536 - rules: - - allow: - method: initialize - - allow: - method: tools/list - - allow: - method: tools/call - deny_rules: - - method: tools/delete - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn write_jsonrpc_default_audit_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_jsonrpc_l7_audit: - name: test_jsonrpc_l7_audit - endpoints: - - host: {host} - port: {port} - path: /rpc - protocol: json-rpc - allowed_ips: - - "10.0.0.0/8" - - "100.64.0.0/10" - - "172.0.0.0/8" - - "198.18.0.0/15" - - "192.168.0.0/16" - - "fc00::/7" - json_rpc: - max_body_bytes: 65536 - rules: - - allow: - method: initialize - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn jsonrpc_l7_enforces_high_level_and_raw_transparent_paths() { - let server = start_test_server(RULES_TEST_SERVER_ALIAS) - .await - .expect("start test server"); - let policy = write_jsonrpc_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import socket -import time -import urllib.error -import urllib.request - -HOST = {host:?} -PORT = {port} -DETAILS = {{ - "debug_target": {{"host": HOST, "port": PORT}}, -}} - -def text(data): - return data.decode(errors="replace") - -def selected_headers(headers): - return {{ - key.lower(): value - for key, value in headers.items() - if key.lower() in ("content-type", "content-length", "server") - }} - -def record_http_error(label, error, request_body): - response_body = error.read() - DETAILS[f"{{label}}_request"] = request_body - DETAILS[f"{{label}}_response"] = {{ - "status": error.code, - "reason": str(error.reason), - "headers": selected_headers(error.headers), - "body": text(response_body), - }} - return error.code - -def post_jsonrpc(label, method, params=None, req_id=1): - body = {{"jsonrpc": "2.0", "id": req_id, "method": method}} - if params is not None: - body["params"] = params - encoded = json.dumps(body).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, body) - -def post_jsonrpc_batch(label, requests): - encoded = json.dumps(requests).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, requests) - -def post_invalid_json(label): - encoded = b"not valid json {{" - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json", "Content-Length": str(len(encoded))}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, text(encoded)) - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: malformed HTTP response: {{response!r}}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error - -def record_raw_response(label, response, body=b""): - code = status_code(response, label) - if code != 200: - DETAILS[f"{{label}}_raw"] = text(response) - if body: - DETAILS[f"{{label}}_body"] = text(body) - return code - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response, body = read_response(sock) - return record_raw_response(f"{{label}}_response", response, body) - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{{label}}_attempt_{{attempt + 1}}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") - -def raw_jsonrpc_status(method, params, label): - target = f"{{HOST}}:{{PORT}}" - body = {{"jsonrpc": "2.0", "id": 1, "method": method}} - if params is not None: - body["params"] = params - encoded = json.dumps(body).encode() - request = ( - f"POST /rpc HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {{len(encoded)}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + encoded - return raw_http_status(label, request) - -results = {{ - # The broad REST endpoint still permits requests outside the RPC route. - "rest_route_allowed": raw_http_status("rest_route_allowed", ( - f"POST /api/status HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - f"Content-Length: 0\r\nConnection: close\r\n\r\n" - ).encode()), - # forward proxy — method-only allow rules - "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}), - "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), - - # forward proxy — method allow/deny rules - "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {{"name": "read_status"}}), - "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}), - "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {{"name": "purge_cache"}}), - "forward_unlisted_method_denied": post_jsonrpc("forward_unlisted_method_denied", "admin.delete"), - - # forward proxy — batch: all requests allowed - "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ - {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, - {{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {{"name": "read_status"}}}}, - ]), - - # forward proxy — batch: one denied request causes full batch denial - "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ - {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, - {{"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {{"name": "purge_cache"}}}}, - ]), - - # forward proxy — invalid JSON body fails closed before generic rules apply - "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), - - # raw socket path — representative allowed and denied cases - "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "raw_method_initialize_allowed"), - "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), - "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {{"name": "read_status"}}, "raw_method_tools_call_allowed"), - "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "raw_method_tools_call_with_unmatched_params_allowed"), - "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "raw_method_tools_delete_denied"), - "raw_unlisted_method_denied": raw_jsonrpc_status("admin.delete", None, "raw_unlisted_method_denied"), -}} -results.update(DETAILS) -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - ("rest_route_allowed", 200), - // forward proxy — allowed - ("forward_method_initialize_allowed", 200), - ("forward_method_tools_list_allowed", 200), - ("forward_method_tools_call_allowed", 200), - ( - "forward_method_tools_call_with_unmatched_params_allowed", - 200, - ), - // forward proxy — method denied - ("forward_method_tools_delete_denied", 403), - ("forward_unlisted_method_denied", 403), - // forward proxy — batch - ("forward_batch_all_allowed", 200), - ("forward_batch_one_denied", 403), - // forward proxy — parse error - ("forward_invalid_json_denied", 403), - // raw socket path — allowed - ("raw_method_initialize_allowed", 200), - ("raw_method_tools_list_allowed", 200), - ("raw_method_tools_call_allowed", 200), - ("raw_method_tools_call_with_unmatched_params_allowed", 200), - // raw socket path — method denied - ("raw_method_tools_delete_denied", 403), - ("raw_unlisted_method_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} - -#[tokio::test] -async fn jsonrpc_forward_proxy_hard_denies_response_frames_in_default_audit_mode() { - let server = start_test_server(AUDIT_TEST_SERVER_ALIAS) - .await - .expect("start test server"); - let policy = - write_jsonrpc_default_audit_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import urllib.error -import urllib.request - -HOST = {host:?} -PORT = {port} - -def post_jsonrpc(body): - encoded = json.dumps(body).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -results = {{ - "forward_unknown_method_audited": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}}), - "forward_response_frame_hard_denied": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "result": {{}}}}), -}} -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - ("forward_unknown_method_audited", 200), - ("forward_response_frame_hard_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs deleted file mode 100644 index f274f20e93..0000000000 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ /dev/null @@ -1,239 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Regression tests: the forward proxy path must evaluate L7 rules for -//! endpoints that have them configured. Allowed requests (e.g. GET on a -//! read-only endpoint) should succeed; denied requests (e.g. POST) should -//! receive 403. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -async fn start_test_server(alias: &str) -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - if self.headers.get("Upgrade"): - self.send_response(101) - self.send_header("Connection", "Upgrade") - self.send_header("Upgrade", "websocket") - self.send_header("Sec-WebSocket-Accept", "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=") - self.end_headers() - return - self.send_response(200) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def do_DELETE(self): - self.send_response(200) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(alias, script).await -} - -fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_l7: - name: test_l7 - endpoints: - - host: {host} - port: {port} - protocol: rest - enforcement: enforce - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - rules: - - allow: - method: GET - path: /allowed - binaries: - - path: /usr/bin/curl - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -/// GET /allowed should succeed — the L7 policy explicitly allows it. -#[tokio::test] -async fn forward_proxy_allows_l7_permitted_request() { - let server = start_test_server("rest-l7-allow.openshell.test") - .await - .expect("start test server"); - let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import urllib.request, urllib.error, json, sys -url = "http://{host}:{port}/allowed" -try: - resp = urllib.request.urlopen(url, timeout=15) - last = {{"status": resp.status, "error": None}} -except urllib.error.HTTPError as e: - last = {{"status": e.code, "error": str(e)}} -except Exception as e: - last = {{"status": -1, "error": str(e)}} -print(json.dumps(last)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - // L7 policy allows GET /allowed — should succeed. - assert!( - guard.create_output.contains("\"status\": 200"), - "expected 200 for L7-allowed GET, got:\n{}", - guard.create_output - ); -} - -/// POST /allowed should be denied — the L7 policy only allows GET. -#[tokio::test] -async fn forward_proxy_denies_l7_blocked_request() { - let server = start_test_server("rest-l7-deny.openshell.test") - .await - .expect("start test server"); - let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import urllib.request, urllib.error, json, sys -url = "http://{host}:{port}/allowed" -req = urllib.request.Request(url, data=b"test", method="POST") -try: - resp = urllib.request.urlopen(req, timeout=15) - print(json.dumps({{"status": resp.status, "error": None}})) -except urllib.error.HTTPError as e: - print(json.dumps({{"status": e.code, "error": str(e)}})) -except Exception as e: - print(json.dumps({{"status": -1, "error": str(e)}})) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - // L7 policy denies POST — should return 403. - assert!( - guard.create_output.contains("\"status\": 403"), - "expected 403 for L7-denied POST, got:\n{}", - guard.create_output - ); -} - -/// A permitted GET cannot switch a REST endpoint into an uninspected tunnel. -#[tokio::test] -async fn rest_upgrade_is_refused_in_enforce_and_audit_modes() { - let server = start_test_server("rest-upgrade.openshell.test") - .await - .expect("start backend"); - for enforcement in ["enforce", "audit"] { - let source = write_policy_with_l7_rules(&server.host, server.port).unwrap(); - let policy = std::fs::read_to_string(source.path()).unwrap().replace( - "enforcement: enforce", - &format!("enforcement: {enforcement}"), - ); - let mut file = NamedTempFile::new().unwrap(); - file.write_all(policy.as_bytes()).unwrap(); - file.flush().unwrap(); - let policy_path = file.path().to_str().unwrap(); - let script = format!( - r#" -import socket -for target in ["/allowed", "http://{host}:{port}/allowed"]: - s = socket.create_connection(("{host}", {port}), timeout=15) - request = ("GET " + target + " HTTP/1.1\r\nHost: {host}:{port}\r\n" - "Connection: Upgrade\r\nUpgrade: websocket\r\n" - "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n") - s.sendall(request.encode()) - response = b"" - while True: - chunk = s.recv(4096) - if not chunk: - break - response += chunk - s.close() - assert response.startswith(b"HTTP/1.1 403"), response - assert b"unsupported_l7_protocol" in response, response - assert b"REST endpoints" in response, response -print("REST_UPGRADE_REFUSED") -"#, - host = server.host, - port = server.port - ); - let guard = - SandboxGuard::create(&["--policy", policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("REST_UPGRADE_REFUSED"), - "{enforcement}: {}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/host_gateway_alias.rs b/e2e/rust/tests/host_gateway_alias.rs index e3f8cb1c9f..98f6ec8159 100644 --- a/e2e/rust/tests/host_gateway_alias.rs +++ b/e2e/rust/tests/host_gateway_alias.rs @@ -4,76 +4,14 @@ #![cfg(feature = "e2e")] use std::io::Write; -use std::process::Stdio; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::is_e2e_driver; use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; +use tempfile::NamedTempFile; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::task::JoinHandle; -const BINDING_PROVIDER_A_NAME: &str = "e2e-static-endpoint-binding-provider-a"; -const BINDING_PROVIDER_B_NAME: &str = "e2e-static-endpoint-binding-provider-b"; -const BINDING_PROFILE_A_ID: &str = "e2e-static-endpoint-binding-a"; -const BINDING_PROFILE_B_ID: &str = "e2e-static-endpoint-binding-b"; - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|e| format!("failed to spawn openshell {}: {e}", args.join(" ")))?; - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn wait_for_sandbox_logs( - sandbox_name: &str, - expected: impl Fn(&str) -> bool, -) -> Result { - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); - - loop { - let logs = run_cli(&[ - "logs", - sandbox_name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) - .await?; - if expected(&logs) { - return Ok(logs); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "timed out waiting for expected sandbox logs:\n{logs}" - )); - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } -} - struct HostServer { port: u16, task: JoinHandle<()>, @@ -81,13 +19,6 @@ struct HostServer { impl HostServer { async fn start(response_body: &str) -> Result { - Self::start_with_auth_check(response_body, None).await - } - - async fn start_with_auth_check( - response_body: &str, - expected_authorization: Option<&str>, - ) -> Result { let listener = TcpListener::bind(("0.0.0.0", 0)) .await .map_err(|e| format!("bind host test server: {e}"))?; @@ -96,14 +27,12 @@ impl HostServer { .map_err(|e| format!("read host test server address: {e}"))? .port(); let response_body = response_body.as_bytes().to_vec(); - let expected_authorization = expected_authorization.map(str::to_string); let task = tokio::spawn(async move { loop { let Ok((mut stream, _)) = listener.accept().await else { break; }; let body = response_body.clone(); - let expected_authorization = expected_authorization.clone(); tokio::spawn(async move { let mut request = Vec::new(); let mut buf = [0_u8; 1024]; @@ -119,17 +48,6 @@ impl HostServer { break; } } - - let body = expected_authorization.map_or(body, |expected| { - let request = String::from_utf8_lossy(&request); - format!( - r#"{{"authorized":{}}}"#, - request.lines().any(|line| { - line.eq_ignore_ascii_case(&format!("Authorization: {expected}")) - }) - ) - .into_bytes() - }); let response = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() @@ -147,114 +65,11 @@ impl HostServer { } } -fn write_binding_profile( - id: &str, - display_name: &str, - env_var: &str, - host: &str, - port: u16, -) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|e| format!("create provider profile: {e}"))?; - let profile = format!( - r#"id: {id} -display_name: {display_name} -category: other -credentials: - - name: bound_token - env_vars: [{env_var}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {host} - port: {port} - path: /allowed/** - protocol: rest - access: full - enforcement: enforce -binaries: [/usr/bin/bash] -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|e| format!("write provider profile: {e}"))?; - file.flush() - .map_err(|e| format!("flush provider profile: {e}"))?; - Ok(file) -} - -fn write_binding_policy(port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create binding policy: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/bin, /usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - binding_test: - name: binding_test - endpoints: - - host: host.openshell.internal - port: {port} - path: /** - protocol: rest - access: full - enforcement: enforce - - host: host.docker.internal - port: {port} - path: /** - protocol: rest - access: full - enforcement: enforce - binaries: - - path: /usr/bin/bash -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write binding policy: {e}"))?; - file.flush() - .map_err(|e| format!("flush binding policy: {e}"))?; - Ok(file) -} - impl Drop for HostServer { fn drop(&mut self) { self.task.abort(); } } - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("profile") - .arg("delete") - .arg(id) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - fn write_policy(port: u16) -> Result { let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; let policy = format!( @@ -412,157 +227,3 @@ printf 'RESPONSE_EOF\n' sandbox.cleanup().await; } } - -#[tokio::test] -async fn static_provider_credentials_are_bound_to_profile_endpoints() { - let server = HostServer::start_with_auth_check("", Some("Bearer e2e-bound-secret")) - .await - .expect("start credential echo server"); - let profile_a = write_binding_profile( - BINDING_PROFILE_A_ID, - "E2E static endpoint binding A", - "BOUND_TOKEN_A", - "host.openshell.internal", - server.port, - ) - .expect("write provider A binding profile"); - let profile_b = write_binding_profile( - BINDING_PROFILE_B_ID, - "E2E static endpoint binding B", - "BOUND_TOKEN_B", - "host.docker.internal", - server.port, - ) - .expect("write provider B binding profile"); - let policy = write_binding_policy(server.port).expect("write binding policy"); - let profile_a_path = profile_a.path().to_string_lossy().into_owned(); - let profile_b_path = profile_b.path().to_string_lossy().into_owned(); - let policy_path = policy.path().to_string_lossy().into_owned(); - - delete_provider(BINDING_PROVIDER_A_NAME).await; - delete_provider(BINDING_PROVIDER_B_NAME).await; - delete_provider_profile(BINDING_PROFILE_A_ID).await; - delete_provider_profile(BINDING_PROFILE_B_ID).await; - run_cli(&["profile", "import", "--file", &profile_a_path]) - .await - .expect("import provider A endpoint-binding profile"); - run_cli(&["profile", "import", "--file", &profile_b_path]) - .await - .expect("import provider B endpoint-binding profile"); - run_cli(&[ - "provider", - "create", - "--name", - BINDING_PROVIDER_A_NAME, - "--type", - BINDING_PROFILE_A_ID, - "--credential", - "BOUND_TOKEN_A=e2e-bound-secret", - ]) - .await - .expect("create endpoint-bound provider A"); - run_cli(&[ - "provider", - "create", - "--name", - BINDING_PROVIDER_B_NAME, - "--type", - BINDING_PROFILE_B_ID, - "--credential", - "BOUND_TOKEN_B=e2e-provider-b-secret", - ]) - .await - .expect("create endpoint-bound provider B"); - - let command = format!( - r#" -http_request() {{ - local host="$1" path="$2" status_line line - HTTP_STATUS= HTTP_BODY= - if ! exec 3<>"/dev/tcp/$host/{port}"; then - HTTP_STATUS=connect-denied - return 1 - fi - printf 'GET %s HTTP/1.1\r\nHost: %s:{port}\r\nAuthorization: Bearer %s\r\nConnection: close\r\n\r\n' "$path" "$host" "$BOUND_TOKEN_A" >&3 - IFS= read -r status_line <&3 || return 1 - status_line="${{status_line%$'\r'}}" - HTTP_STATUS="${{status_line#* }}" - HTTP_STATUS="${{HTTP_STATUS%% *}}" - while IFS= read -r line <&3; do - line="${{line%$'\r'}}" - [[ -z "$line" ]] && break - done - while IFS= read -r line <&3 || [[ -n "$line" ]]; do HTTP_BODY+="$line"; done - exec 3>&- 3<&- -}} -http_request host.openshell.internal /allowed/check; allowed="$HTTP_BODY" -http_request host.docker.internal /allowed/check || true; host_denied="$HTTP_STATUS" -http_request host.openshell.internal /other/check; path_denied="$HTTP_STATUS" -printf 'ALLOWED=%s HOST_DENIED=%s PATH_DENIED=%s\n' "$allowed" "$host_denied" "$path_denied" -"#, - port = server.port, - ); - let mut guard = SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - BINDING_PROVIDER_A_NAME, - "--provider", - BINDING_PROVIDER_B_NAME, - "--no-auto-providers", - "--", - "/usr/bin/bash", - "-c", - &command, - ]) - .await - .expect("run endpoint-binding requests"); - - let logs = wait_for_sandbox_logs(&guard.name, |logs| { - logs.contains("openshell.provider_credential.endpoint_mismatch") - && logs.contains("credential_endpoint_mismatch") - }) - .await - .expect("fetch endpoint mismatch logs"); - assert!( - guard - .create_output - .contains(r#"ALLOWED={"authorized":true}"#), - "credential should resolve at the bound endpoint:\n{}\nlogs:\n{logs}", - guard.create_output, - ); - let expected_host_denial = if is_e2e_driver("podman") { - "HOST_DENIED=connect-denied" - } else { - "HOST_DENIED=403" - }; - assert!( - guard.create_output.contains(expected_host_denial), - "same placeholder must be denied at an unbound host:\n{}", - guard.create_output - ); - assert!( - guard.create_output.contains("PATH_DENIED=403"), - "same placeholder must be denied at an unbound path on its bound host:\n{}", - guard.create_output - ); - - assert!( - logs.contains("openshell.provider_credential.endpoint_mismatch") - && logs.contains("credential_endpoint_mismatch"), - "OCSF logs should explain the endpoint-binding denial without secret material:\n{logs}" - ); - assert!( - !logs.contains("e2e-bound-secret") - && !logs.contains("e2e-provider-b-secret") - && !logs.contains("BOUND_TOKEN_A") - && !logs.contains("BOUND_TOKEN_B"), - "OCSF logs must not contain credential values or environment keys:\n{logs}" - ); - - guard.cleanup().await; - delete_provider(BINDING_PROVIDER_A_NAME).await; - delete_provider(BINDING_PROVIDER_B_NAME).await; - delete_provider_profile(BINDING_PROFILE_A_ID).await; - delete_provider_profile(BINDING_PROFILE_B_ID).await; -} diff --git a/e2e/rust/tests/listener_lifetime.rs b/e2e/rust/tests/listener_lifetime.rs deleted file mode 100644 index ec58a0dcd2..0000000000 --- a/e2e/rust/tests/listener_lifetime.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -const LIFETIME_PROBE: &str = r" -import errno, json, os, socket, subprocess, sys - -def assert_closed(address, inode): - # Check before bind can trigger the broker's stale-descriptor collection. - for table in ('/proc/net/tcp', '/proc/net/tcp6'): - for line in open(table): - fields = line.split() - assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(inode)), line - with socket.socket() as client: - client.settimeout(2) - assert client.connect_ex(tuple(address)) == errno.ECONNREFUSED, address - with socket.socket() as replacement: - replacement.bind(tuple(address)) - -def inode(sock): - return int(os.readlink('/proc/self/fd/' + str(sock.fileno()))[8:-1]) - -with socket.socket() as bound: - bound.bind(('127.0.0.1', 0)) - address = bound.getsockname() -with socket.socket() as client: - client.settimeout(2) - assert client.connect_ex(address) == errno.ECONNREFUSED - -for host in ('127.0.0.1', '::1'): - family = socket.AF_INET if host == '127.0.0.1' else socket.AF_INET6 - listener = socket.socket(family) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind((host, 0)) - listener.listen(1) - listener.listen(8) - address, identity = listener.getsockname(), inode(listener) - for operation, expected in ((lambda: listener.bind(address), errno.EINVAL), - (lambda: listener.connect(address), errno.EISCONN)): - try: - operation() - raise AssertionError('operation on listener unexpectedly succeeded') - except OSError as error: - assert error.errno == expected, error - alias = listener.dup() - alias.settimeout(2) - listener.close() - with socket.socket(family) as client: - client.settimeout(2) - client.connect(address) - accepted, _ = alias.accept() - with accepted: - accepted.sendall(b'ok') - assert client.recv(2) == b'ok' - alias.close() - table = '/proc/net/tcp' if family == socket.AF_INET else '/proc/net/tcp6' - for line in open(table): - fields = line.split() - assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(identity)), line - with socket.socket(family) as client: - client.settimeout(2) - assert client.connect_ex(address) == errno.ECONNREFUSED - with socket.socket(family) as replacement: - replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - replacement.bind(address) - -# The child exits while still owning its listener. Its broker must not keep it alive. -child = r''' -import json, os, socket -s = socket.socket() -s.bind(('127.0.0.1', 0)) -s.listen() -inode = int(os.readlink('/proc/self/fd/' + str(s.fileno()))[8:-1]) -print(json.dumps([s.getsockname(), inode]), flush=True) -os._exit(0) -''' -result = subprocess.run([sys.executable, '-c', child], check=True, capture_output=True, text=True, timeout=10) -address, identity = json.loads(result.stdout) -assert_closed(address, identity) -print(json.dumps([address, identity])) -"; - -#[tokio::test] -async fn closing_or_exiting_listener_refuses_connections_and_releases_port() { - let mut sandbox = SandboxGuard::create(&[]).await.expect("create sandbox"); - let output = sandbox - .exec(&["python3", "-c", LIFETIME_PROBE]) - .await - .expect("listener lifetime probe"); - let observer = r" -import json, os, sys -_, inode = json.loads(sys.argv[1]) -for fd in os.listdir('/proc/self/fd'): - try: - target = os.readlink('/proc/self/fd/' + fd) - except FileNotFoundError: - continue - assert target != 'socket:[' + str(inode) + ']', (fd, target) -"; - sandbox - .exec(&["python3", "-c", observer, output.trim()]) - .await - .expect("subsequent exec must not inherit closed listener"); - sandbox.cleanup().await; -} - -#[tokio::test] -async fn unrelated_exec_does_not_inherit_broker_preconnect_socket() { - // Request a non-CLOEXEC socket explicitly; Python's socket constructor - // otherwise masks broker inheritance bugs by requesting SOCK_CLOEXEC. - let owner = r" -import ctypes, json, os, socket, time -libc = ctypes.CDLL(None, use_errno=True) -fd = libc.socket(socket.AF_INET, socket.SOCK_STREAM, 0) -assert fd >= 0, ctypes.get_errno() -assert os.get_inheritable(fd) -inode = int(os.readlink('/proc/self/fd/' + str(fd))[8:-1]) -with open('/sandbox/preconnect-inode.json', 'w') as f: - json.dump(inode, f) -while True: - time.sleep(60) -"; - let mut sandbox = SandboxGuard::create_detached_main(&["python3", "-c", owner]) - .await - .expect("create socket owner"); - let observer = r" -import json, os, time -path = '/sandbox/preconnect-inode.json' -for _ in range(100): - try: - with open(path) as f: - inode = json.load(f) - break - except (FileNotFoundError, json.JSONDecodeError): - time.sleep(0.1) -else: - raise AssertionError('socket owner did not publish its inode') -for fd in os.listdir('/proc/self/fd'): - try: - target = os.readlink('/proc/self/fd/' + fd) - except FileNotFoundError: - continue - assert target != 'socket:[' + str(inode) + ']', (fd, target) -print('broker socket was not inherited') -"; - sandbox - .exec(&["python3", "-c", observer]) - .await - .expect("unrelated exec inherited broker descriptor"); - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs deleted file mode 100644 index b6a35c8898..0000000000 --- a/e2e/rust/tests/live_policy_update.rs +++ /dev/null @@ -1,704 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for live policy updates on a running sandbox. -//! -//! Covers the full round-trip: -//! - Create sandbox with policy A -//! - Verify initial policy version via `policy get` -//! - Push same policy A again -> no version bump (idempotent) -//! - Push different policy B -> new version, `--wait` for sandbox to load it -//! - Verify policy history via `policy list` -//! -//! These tests replace the Python e2e tests `test_live_policy_update_and_logs` -//! and `test_live_policy_update_from_empty_network_policies`, which were flaky -//! due to hard-coded 90s poll timeouts. The Rust tests use the CLI's built-in -//! `--wait` flag for reliable synchronization. -//! -//! Note: the removed Python tests also covered `GetSandboxLogs` RPC and -//! verified actual proxy connectivity after policy update. Those are tracked -//! as follow-up coverage gaps -- the proxy enforcement path is covered by the -//! existing L4/L7/SSRF Python e2e tests, and log fetching needs a dedicated -//! test. - -#![cfg(feature = "e2e")] - -use std::fmt::Write as _; -use std::io::Write; -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SPARSE_POLICY: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/policy-advisor/sandbox-policy.yaml" -)); - -// --------------------------------------------------------------------------- -// Policy YAML builders -// --------------------------------------------------------------------------- - -/// Build a policy YAML that allows any binary to reach the given hosts on -/// port 443. Keep its filesystem paths aligned with the empty-network policy -/// so live network updates do not remove startup filesystem access. -/// -/// NOTE: The indentation in the format string is load-bearing YAML structure. -fn write_policy(hosts: &[&str]) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - - let mut network_rules = String::new(); - for (i, host) in hosts.iter().enumerate() { - let _ = write!( - network_rules, - r#" rule_{i}: - name: rule_{i} - endpoints: - - host: {host} - port: 443 - binaries: - - path: "/**" -"# - ); - } - - let policy = format!( - r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /bin - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -network_policies: -{network_rules}" - ); - - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -/// Build a minimal policy YAML with no network rules. Both /bin and /usr -/// are readable so shell entrypoints work on merged and unmerged images. -fn write_empty_network_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - - let policy = r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /bin - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort -"; - - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn write_sparse_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - file.write_all(SPARSE_POLICY.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -// --------------------------------------------------------------------------- -// CLI helpers -// --------------------------------------------------------------------------- - -struct CliResult { - success: bool, - output: String, - exit_code: Option, -} - -/// Run an `openshell` CLI command and return the result. -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = strip_ansi(&format!("{stdout}{stderr}")); - - CliResult { - success: output.status.success(), - output: combined, - exit_code: output.status.code(), - } -} - -/// Extract the policy version number from `policy get` output. -/// -/// Uses the shared `extract_field` helper to find `Version: ` or -/// `Revision: ` in CLI tabular output. -fn extract_version(output: &str) -> Option { - extract_field(output, "Version") - .or_else(|| extract_field(output, "Revision")) - .and_then(|v| v.parse::().ok()) -} - -/// Extract the policy hash from `policy get` output. -fn extract_hash(output: &str) -> Option { - extract_field(output, "Hash").or_else(|| extract_field(output, "Policy hash")) -} - -/// Check that a version number appears in `policy list` output as a -/// distinct field value (not just a substring of some other number). -/// -/// Looks for the version number preceded by whitespace or at the start -/// of a line, to avoid matching "2" inside "12" or timestamps. -fn list_output_contains_version(output: &str, version: u32) -> bool { - let v = version.to_string(); - output.lines().any(|line| { - line.split_whitespace() - .any(|word| word == v || word.starts_with(&format!("{v} "))) - }) -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -/// Read the effective policy through the same CLI boundary used by operators. -async fn l7_scope_snapshot(name: &str) -> serde_json::Value { - let result = run_cli(&["policy", "get", name, "--full", "--output", "json"]).await; - assert!(result.success, "policy snapshot failed: {}", result.output); - let snapshot: serde_json::Value = - serde_json::from_str(&result.output).expect("policy get returns JSON"); - assert!(snapshot["version"].as_u64().is_some()); - assert!( - snapshot["hash"] - .as_str() - .is_some_and(|hash| !hash.is_empty()) - ); - assert!(snapshot["policy"]["network_policies"].is_object()); - snapshot -} - -/// Incomplete declarations reject without a revision; an explicit target -/// changes only its endpoint even when another rule shares the host and ports. -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn l7_append_target_scope_round_trip() { - let mut policy = write_empty_network_policy().expect("write base policy"); - policy - .write_all( - br" -network_policies: - internal_api: - name: internal_api - binaries: - - path: /usr/bin/curl - - path: /usr/bin/python3 - endpoints: - - host: api.example.com - ports: [443, 8443] - protocol: rest - access: read-only - - host: other.example.com - port: 443 - protocol: rest - access: read-only - sibling: - name: sibling - binaries: - - path: /usr/bin/wget - endpoints: - - host: api.example.com - ports: [443, 8443] - protocol: rest - access: read-only -", - ) - .expect("write scoped policy"); - policy.flush().expect("flush scoped policy"); - let path = policy.path().to_str().expect("UTF-8 policy path"); - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create scoped-policy sandbox"); - let before = l7_scope_snapshot(&guard.name).await; - - // Check each independent axis with the other fully declared, so a guard - // requiring both mismatches at once cannot satisfy this regression. - for (ports, binaries) in [ - ( - "api.example.com:443:POST:/admin", - vec!["/usr/bin/curl", "/usr/bin/python3"], - ), - ( - "api.example.com:443,8443:POST:/admin", - vec!["/usr/bin/curl"], - ), - ] { - let mut args = vec![ - "policy", - "update", - &guard.name, - "--rule-name", - "internal_api", - "--add-allow", - ports, - ]; - for binary in binaries { - args.extend(["--binary", binary]); - } - let rejected = run_cli(&args).await; - assert!( - !rejected.success, - "partial scope was accepted: {}", - rejected.output - ); - let unchanged = l7_scope_snapshot(&guard.name).await; - for field in ["version", "hash", "policy"] { - assert_eq!(unchanged[field], before[field], "rejection changed {field}"); - } - } - - let common = [ - "policy", - "update", - &guard.name, - "--rule-name", - "internal_api", - "--binary", - "/usr/bin/curl", - "--binary", - "/usr/bin/python3", - ]; - let mut allow_args = common.to_vec(); - allow_args.extend([ - "--add-allow", - "api.example.com:443,8443:POST:/admin", - "--wait", - ]); - let accepted = run_cli(&allow_args).await; - assert!( - accepted.success, - "explicit allow failed: {}", - accepted.output - ); - let after_allow = l7_scope_snapshot(&guard.name).await; - assert!(after_allow["version"].as_u64() > before["version"].as_u64()); - assert_ne!(after_allow["hash"], before["hash"]); - let rules = &after_allow["policy"]["network_policies"]; - assert_eq!( - rules["sibling"], - before["policy"]["network_policies"]["sibling"] - ); - assert_eq!( - rules["internal_api"]["endpoints"][1], - before["policy"]["network_policies"]["internal_api"]["endpoints"][1] - ); - assert!( - rules["internal_api"]["endpoints"][0]["rules"] - .as_array() - .expect("allow rules") - .iter() - .any(|rule| rule["allow"]["method"] == "POST" && rule["allow"]["path"] == "/admin") - ); - - let mut deny_args = common.to_vec(); - deny_args.extend([ - "--add-deny", - "api.example.com:443,8443:POST:/admin/private", - "--wait", - ]); - let denied = run_cli(&deny_args).await; - assert!(denied.success, "explicit deny failed: {}", denied.output); - let after_deny = l7_scope_snapshot(&guard.name).await; - assert!(after_deny["version"].as_u64() > after_allow["version"].as_u64()); - assert_eq!( - after_deny["policy"]["network_policies"]["sibling"], - before["policy"]["network_policies"]["sibling"] - ); - assert!( - after_deny["policy"]["network_policies"]["internal_api"]["endpoints"][0]["deny_rules"] - .as_array() - .expect("deny rules") - .iter() - .any(|rule| rule["method"] == "POST" && rule["path"] == "/admin/private") - ); - guard.cleanup().await; -} - -/// Test the full live policy update lifecycle: -/// -/// 1. Create sandbox with policy A and `--keep` -/// 2. Verify initial version >= 1 -/// 3. Push same policy A -> version unchanged (idempotent) -/// 4. Push policy B (adds example.com) with `--wait` -> new version -/// 5. Push policy B again -> idempotent -/// 6. Verify policy list shows both versions -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn live_policy_update_round_trip() { - // --- Write two distinct policy files --- - let policy_a = write_policy(&["api.anthropic.com"]).expect("write policy A"); - let policy_b = write_policy(&["api.anthropic.com", "example.com"]).expect("write policy B"); - - let policy_a_path = policy_a - .path() - .to_str() - .expect("policy A path should be utf-8") - .to_string(); - let policy_b_path = policy_b - .path() - .to_str() - .expect("policy B path should be utf-8") - .to_string(); - - // --- Create a long-running sandbox with its startup-only policy fields --- - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_a_path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with policy A"); - - // --- Verify initial policy version --- - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let initial_version = extract_version(&r.output).unwrap_or_else(|| { - panic!( - "could not parse version from policy get output:\n{}", - r.output - ) - }); - assert!( - initial_version >= 1, - "initial policy version should be >= 1, got {initial_version}" - ); - - let initial_hash = extract_hash(&r.output); - - // --- Push same policy A again -> should be idempotent --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_a_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set A (repeat) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after repeat should succeed:\n{}", - r.output - ); - - let repeat_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after repeat:\n{}", r.output)); - assert_eq!( - repeat_version, initial_version, - "same policy should not bump version: expected {initial_version}, got {repeat_version}" - ); - - if let (Some(ih), Some(rh)) = (&initial_hash, &extract_hash(&r.output)) { - assert_eq!(ih, rh, "same policy should produce same hash"); - } - - // --- Push policy B -> should create new version --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set B should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after B should succeed:\n{}", - r.output - ); - - let new_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after B:\n{}", r.output)); - assert!( - new_version > initial_version, - "different policy should bump version: expected > {initial_version}, got {new_version}" - ); - - if let (Some(ih), Some(nh)) = (&initial_hash, &extract_hash(&r.output)) { - assert_ne!(ih, nh, "different policy should produce different hash"); - } - - // --- Push policy B again -> idempotent --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set B (repeat) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after B repeat should succeed:\n{}", - r.output - ); - - let repeat_b_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after B repeat:\n{}", r.output)); - assert_eq!( - repeat_b_version, new_version, - "same policy B should not bump version: expected {new_version}, got {repeat_b_version}" - ); - - // --- Verify policy list shows revision history --- - let r = run_cli(&["policy", "list", &guard.name]).await; - assert!( - r.success, - "policy list should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - // Both versions should appear in the list output. - assert!( - list_output_contains_version(&r.output, new_version), - "policy list should contain version {new_version}:\n{}", - r.output - ); - assert!( - list_output_contains_version(&r.output, initial_version), - "policy list should contain initial version {initial_version}:\n{}", - r.output - ); - - guard.cleanup().await; -} - -/// Test live policy update from an initially empty network policy: -/// -/// 1. Create sandbox with no network rules and `--keep` -/// 2. Push policy with a network rule using `--wait` -/// 3. Verify the version bumped -#[tokio::test] -async fn live_policy_update_from_empty_network_policies() { - let empty_policy = write_empty_network_policy().expect("write empty network policy"); - let full_policy = write_policy(&["example.com"]).expect("write full policy"); - - let empty_path = empty_policy - .path() - .to_str() - .expect("empty policy path should be utf-8") - .to_string(); - let full_path = full_policy - .path() - .to_str() - .expect("full policy path should be utf-8") - .to_string(); - - // Create the sandbox with the empty network policy so subsequent live - // updates retain the same startup-only filesystem, landlock, and process - // fields. - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &empty_path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with empty network policy"); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get (empty) should succeed:\n{}", - r.output - ); - - let initial_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version from empty policy:\n{}", r.output)); - - // Push policy with network rules. - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &full_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set (full) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!(r.success, "policy get (full) should succeed:\n{}", r.output); - - let new_version = extract_version(&r.output).unwrap_or_else(|| { - panic!( - "could not parse version after adding network rules:\n{}", - r.output - ) - }); - assert!( - new_version > initial_version, - "adding network rules should create new version > {initial_version}, got {new_version}" - ); - - guard.cleanup().await; -} - -/// Regression for #2159: a sparse initial policy that the supervisor enriches -/// with baseline filesystem paths during startup must have its resulting -/// revision acknowledged as loaded, not left `Pending`. -/// -/// Reproduction (matches the maintainer's triage): create a sandbox with the -/// network-only `examples/policy-advisor/sandbox-policy.yaml`. The supervisor -/// adds baseline filesystem paths, syncs the enriched policy back to the -/// gateway (creating revision 2, superseding revision 1), builds the OPA engine -/// and then acknowledges revision 2 as loaded. Before the fix, revision 2 -/// stayed `Pending` even though the sandbox was `Ready` and the policy was -/// effective. -/// -/// NOTE: This exercises the Docker-backed supervisor built from this branch. -/// The exact `policy list` status wording ("Loaded"/"Superseded") may differ by -/// CLI version; the assertions below key on the effective version reaching 2 and -/// no revision remaining `Pending` once the acknowledgement lands. -#[tokio::test] -async fn initial_sparse_policy_is_acknowledged_as_loaded() { - let sparse_policy = write_sparse_policy().expect("write sparse policy fixture"); - let sparse_policy_path = sparse_policy - .path() - .to_str() - .expect("sparse policy path is not UTF-8"); - - let mut guard = SandboxGuard::create_keep_with_args( - &[ - "--name", - "e2e-sparse-enrich", - "--policy", - sparse_policy_path, - "--no-tty", - ], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with sparse policy"); - - // The enriched revision (2) is synced during startup; the acknowledgement - // (LOADED) is delivered by the supervisor's poll loop shortly after Ready. - // Poll until the effective policy is version 2 and no revision is Pending. - let mut acknowledged = false; - let mut last_list = String::new(); - for _ in 0..30 { - let get = run_cli(&["policy", "get", &guard.name]).await; - let version = get.success.then(|| extract_version(&get.output)).flatten(); - - let list = run_cli(&["policy", "list", &guard.name]).await; - last_list = list.output.clone(); - let pending = list.output.to_lowercase().contains("pending"); - - if version == Some(2) && list.success && !pending { - acknowledged = true; - break; - } - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - - assert!( - acknowledged, - "enriched initial policy should reach revision 2 with no Pending revision.\n\ - last `policy list` output:\n{last_list}" - ); - - // Both the superseded original (1) and the loaded enriched revision (2) - // must appear in the revision history. - assert!( - list_output_contains_version(&last_list, 2), - "policy list should contain revision 2:\n{last_list}" - ); - assert!( - list_output_contains_version(&last_list, 1), - "policy list should contain revision 1:\n{last_list}" - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/mcp_sessionless.rs b/e2e/rust/tests/mcp_sessionless.rs deleted file mode 100644 index bf2261efa1..0000000000 --- a/e2e/rust/tests/mcp_sessionless.rs +++ /dev/null @@ -1,381 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! MCP request profiles through the sandbox's transparent network interception. -//! -//! A shared fixture serves legacy initialization, sessionless discovery, named -//! tools, and a bounded subscription stream. Upstream receipts distinguish a -//! policy denial from a tool request accepted by the fixture. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SERVER_ALIAS: &str = "mcp-sessionless.openshell.test"; - -const SERVER_SCRIPT: &str = r#" -import json -from http.server import BaseHTTPRequestHandler, HTTPServer - -SESSIONLESS_VERSION = "2026-07-28" -received = [] - -class Handler(BaseHTTPRequestHandler): - def reply(self, status, payload, content_type="application/json"): - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def do_GET(self): - # The container fixture's readiness probe uses the root URL. - self.reply(200 if self.path == "/" else 405, b"") - - def read_body(self): - if self.headers.get("Transfer-Encoding", "").lower() != "chunked": - return self.rfile.read(int(self.headers.get("Content-Length", "0"))) - body = bytearray() - while True: - size = int(self.rfile.readline().split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - return bytes(body) - body.extend(self.rfile.read(size)) - self.rfile.read(2) - - def do_POST(self): - message = json.loads(self.read_body()) - method = message["method"] - params = message.get("params", {}) - version = (params.get("protocolVersion") if method == "initialize" - else self.headers.get("MCP-Protocol-Version")) - # Record parsed requests before fixture admission so rejected revisions - # and tool attempts stay visible. HTTPServer handles requests serially. - received.append([version, method, params.get("name")]) - if self.path != "/mcp" or version not in SUPPORTED_VERSIONS: - self.reply(400, b"request revision did not reach the fixture intact") - return - if version == SESSIONLESS_VERSION: - meta = params.get("_meta", {}) - if (self.headers.get("MCP-Protocol-Version") != version - or self.headers.get("Mcp-Method") != method - or meta.get("io.modelcontextprotocol/protocolVersion") != version - or meta.get("io.modelcontextprotocol/clientCapabilities") != {} - or (method == "tools/call" - and self.headers.get("Mcp-Name") != params["name"])): - self.reply(400, b"request metadata did not reach the fixture intact") - return - - if method == "initialize" and version != SESSIONLESS_VERSION: - result = { - "protocolVersion": version, - "capabilities": {"tools": {}}, - "serverInfo": {"name": "openshell-profile-fixture", "version": "1"}, - } - elif method == "notifications/initialized" and version != SESSIONLESS_VERSION: - self.reply(202, b"") - return - elif method == "server/discover" and version == SESSIONLESS_VERSION: - result = { - "supportedVersions": [version], - "capabilities": {"tools": {"listChanged": True}}, - "ttlMs": 0, - "cacheScope": "private", - "_meta": { - "io.modelcontextprotocol/serverInfo": { - "name": "openshell-sessionless-fixture", "version": "1" - } - }, - } - elif method == "tools/call": - # Both tool names work upstream; OpenShell owns the policy denial. - result = { - "content": [{"type": "text", "text": params["name"]}], - "isError": False, - "_meta": {"fixtureRequests": list(received)}, - } - if version == SESSIONLESS_VERSION: - result["resultType"] = "complete" - elif method == "subscriptions/listen" and version == SESSIONLESS_VERSION: - subscription_meta = {"io.modelcontextprotocol/subscriptionId": message["id"]} - events = [ - { - "jsonrpc": "2.0", - "method": "notifications/subscriptions/acknowledged", - "params": {"notifications": params["notifications"], "_meta": subscription_meta}, - }, - { - "jsonrpc": "2.0", - "method": "notifications/tools/list_changed", - "params": {"_meta": subscription_meta}, - }, - { - "jsonrpc": "2.0", - "id": message["id"], - "result": {"resultType": "complete", "_meta": subscription_meta}, - }, - ] - body = "".join("event: message\ndata: " + json.dumps(event) + "\n\n" for event in events) - self.reply(200, body.encode(), "text/event-stream") - return - else: - self.reply(400, b"unexpected method for the selected fixture revision") - return - - self.reply(200, json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": result}).encode()) - - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - -const CLIENT_HELPERS: &str = r#" -import json -import urllib.error -import urllib.request -# Direct client connections pass through the sandbox's transparent interception. -opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - -def post(request_id, method, params, version): - params = dict(params) - headers = { - "Content-Type": "application/json", - "Accept": "application/json, text/event-stream", - } - if method != "initialize": - headers["MCP-Protocol-Version"] = version - if version == "2026-07-28": - params["_meta"] = { - "io.modelcontextprotocol/protocolVersion": version, - "io.modelcontextprotocol/clientCapabilities": {}, - "io.modelcontextprotocol/clientInfo": {"name": "openshell-e2e", "version": "1"}, - } - headers["Mcp-Method"] = method - if method == "tools/call": - headers["Mcp-Name"] = params["name"] - message = {"jsonrpc": "2.0", "method": method, "params": params} - if request_id is not None: - message["id"] = request_id - request = urllib.request.Request( - f"http://{HOST}:{PORT}/mcp", - data=json.dumps(message).encode(), - headers=headers, - method="POST", - ) - try: - with opener.open(request, timeout=15) as response: - return response.status, response.headers.get_content_type(), response.read() - except urllib.error.HTTPError as error: - return error.code, error.headers.get_content_type(), error.read() -"#; - -const CLIENT_SCRIPT: &str = r#" -VERSION = "2026-07-28" - -status, content_type, body = post(1, "server/discover", {}, VERSION) -assert status == 200, ("discovery", status, body) -assert content_type == "application/json", content_type -discovery = json.loads(body) -assert discovery["id"] == 1, discovery -assert discovery["result"]["supportedVersions"] == [VERSION], discovery - -status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, VERSION) -assert status == 200, ("allowed tool", status, body) -tool = json.loads(body) -assert tool["id"] == 2, tool -assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool - -status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, VERSION) -assert status == 403, ("denied tool", status, body) - -status, content_type, body = post(4, "subscriptions/listen", {"notifications": {"toolsListChanged": True}}, VERSION) -assert status == 200, ("subscription", status, body) -assert content_type == "text/event-stream", (content_type, body) -events = [json.loads(line[6:]) for line in body.decode().splitlines() if line.startswith("data: ")] -assert len(events) == 3, events -assert events[0]["method"] == "notifications/subscriptions/acknowledged", events -assert events[0]["params"]["notifications"] == {"toolsListChanged": True}, events -assert events[1]["method"] == "notifications/tools/list_changed", events -assert events[1]["params"]["_meta"]["io.modelcontextprotocol/subscriptionId"] == 4, events -assert events[2]["id"] == 4 and events[2]["result"]["resultType"] == "complete", events - -print("MCP_SESSIONLESS_OK discovery=200 allowed_tool=200 denied_tool=403 subscription=200") -"#; - -const PROFILE_CLIENT_SCRIPT: &str = r#" -expected_receipts = [] -for version in SELECTED_VERSIONS: - if version == "2026-07-28": - status, _, body = post(1, "server/discover", {}, version) - assert status == 200, (version, "discovery", status, body) - assert json.loads(body)["result"]["supportedVersions"] == [version], body - expected_receipts.append([version, "server/discover", None]) - else: - status, _, body = post(1, "initialize", { - "protocolVersion": version, - "capabilities": {}, - "clientInfo": {"name": "openshell-e2e", "version": "1"}, - }, version) - assert status == 200, (version, "initialize", status, body) - assert json.loads(body)["result"]["protocolVersion"] == version, body - expected_receipts.append([version, "initialize", None]) - status, _, body = post(None, "notifications/initialized", {}, version) - assert status == 202, (version, "initialized", status, body) - expected_receipts.append([version, "notifications/initialized", None]) - - status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, version) - assert status == 200, (version, "allowed tool", status, body) - tool = json.loads(body) - assert tool["id"] == 2, tool - assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool - expected_receipts.append([version, "tools/call", "read_status"]) - assert tool["result"]["_meta"]["fixtureRequests"] == expected_receipts, tool - - status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, version) - assert status == 403, (version, "denied tool", status, body) - - # The fixture accepts both tools. A later allowed call proves the denial - # came from the proxy and no denied operation reached the upstream. - status, _, body = post(4, "tools/call", {"name": "read_status", "arguments": {}}, version) - assert status == 200, (version, "receipt tool", status, body) - receipt = json.loads(body) - assert receipt["id"] == 4, receipt - expected_receipts.append([version, "tools/call", "read_status"]) - assert receipt["result"]["_meta"]["fixtureRequests"] == expected_receipts, receipt - print(f"MCP_PROFILE_OK version={version} allowed_tool=200 denied_tool=403 receipts=verified") -"#; - -async fn start_server(alias: &str, versions: &[&str]) -> Result { - let versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let script = format!("SUPPORTED_VERSIONS = {versions}\n{SERVER_SCRIPT}"); - ContainerHttpServer::start_python(alias, &script).await -} - -fn write_policy(host: &str, port: u16, versions: &[&str]) -> Result { - let mut file = NamedTempFile::new().map_err(|err| format!("create temp policy: {err}"))?; - let legacy_rules = if versions.iter().any(|version| *version != "2026-07-28") { - " - allow:\n method: initialize\n - allow:\n method: notifications/initialized\n" - } else { - "" - }; - let versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let policy = format!( - r#"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: - mcp_sessionless: - name: mcp_sessionless - endpoints: - - host: {host} - port: {port} - path: /mcp - protocol: mcp - enforcement: enforce - allowed_ips: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7"] - mcp: - versions: {versions} - max_body_bytes: 65536 - rules: -{legacy_rules} - allow: - method: server/discover - - allow: - method: tools/call - tool: read_status - - allow: - method: subscriptions/listen - deny_rules: - - method: tools/call - tool: read_details - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|err| format!("write temp policy: {err}"))?; - file.flush() - .map_err(|err| format!("flush temp policy: {err}"))?; - Ok(file) -} - -async fn run_client( - server: &ContainerHttpServer, - versions: &[&str], - client: &str, -) -> Result { - let policy = write_policy(&server.host, server.port, versions)?; - let policy_path = policy - .path() - .to_str() - .ok_or("temp policy path is not UTF-8")?; - let selected_versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let script = format!( - "HOST = {:?}\nPORT = {}\nSELECTED_VERSIONS = {selected_versions}\n{CLIENT_HELPERS}\n{client}", - server.host, server.port - ); - SandboxGuard::create(&["--policy", policy_path, "--", "python3", "-c", &script]).await -} - -#[tokio::test] -async fn sessionless_discovery_tools_and_subscription_use_request_metadata() { - let versions = ["2026-07-28"]; - let server = start_server(SERVER_ALIAS, &versions) - .await - .expect("start sessionless MCP fixture"); - let sandbox = run_client(&server, &versions, CLIENT_SCRIPT) - .await - .expect("run sessionless MCP client in sandbox"); - - assert!( - sandbox.create_output.contains( - "MCP_SESSIONLESS_OK discovery=200 allowed_tool=200 denied_tool=403 subscription=200" - ), - "expected completed sessionless MCP assertions, got:\n{}", - sandbox.create_output - ); -} - -#[tokio::test] -async fn legacy_and_multi_version_profiles_authorize_tools_through_sandbox() { - for versions in [ - &["2025-03-26"][..], - &["2025-06-18"][..], - &["2025-11-25", "2026-07-28"][..], - ] { - // Each scenario starts with fresh upstream receipts. Its distinct alias - // avoids the sessionless test's fixture, and cleanup precedes alias reuse. - let server = start_server("mcp-profiles.openshell.test", versions) - .await - .unwrap_or_else(|err| panic!("{versions:?}: start MCP fixture: {err}")); - let mut sandbox = run_client(&server, versions, PROFILE_CLIENT_SCRIPT) - .await - .unwrap_or_else(|err| panic!("{versions:?}: run MCP sandbox client: {err}")); - for version in versions { - let marker = format!( - "MCP_PROFILE_OK version={version} allowed_tool=200 denied_tool=403 receipts=verified" - ); - assert!( - sandbox.create_output.contains(&marker), - "{versions:?}: expected completed {version} assertions, got:\n{}", - sandbox.create_output - ); - } - sandbox.cleanup().await; - } -} diff --git a/e2e/rust/tests/no_proxy.rs b/e2e/rust/tests/no_proxy.rs deleted file mode 100644 index 447c408f8b..0000000000 --- a/e2e/rust/tests/no_proxy.rs +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -fn localhost_transparent_script() -> &'static str { - r#" -import json -import os -import threading -import urllib.request -from http.server import BaseHTTPRequestHandler, HTTPServer - -for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): - assert name not in os.environ, f'unexpected proxy environment variable: {name}' - -class Handler(BaseHTTPRequestHandler): - def log_message(self, format, *args): - pass - - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(b'{"message":"hello"}') - -server = HTTPServer(('127.0.0.1', 0), Handler) -thread = threading.Thread(target=server.serve_forever, daemon=True) -thread.start() - -try: - with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: - print(json.dumps({ - 'proxy_env_absent': True, - 'payload': json.loads(response.read().decode()), - }), flush=True) -finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() -"# -} - -#[tokio::test] -async fn sandbox_reaches_localhost_without_proxy_environment() { - let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_transparent_script()]) - .await - .expect("sandbox create with transparent localhost check"); - - assert!( - guard - .create_output - .contains(r#"{"proxy_env_absent": true, "payload": {"message": "hello"}}"#), - "expected localhost HTTP request to stay local and succeed:\n{}", - guard.create_output - ); -} diff --git a/e2e/rust/tests/port_forward.rs b/e2e/rust/tests/port_forward.rs deleted file mode 100644 index e91919d177..0000000000 --- a/e2e/rust/tests/port_forward.rs +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: TCP port forwarding through a sandbox. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::time::Duration; - -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; - -use openshell_e2e::harness::port::{find_free_port, wait_for_port}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// Python script that runs a single-threaded TCP echo server inside the -/// sandbox. It prints `echo-server-ready` to stdout once listening, which -/// the harness uses as the readiness marker. -fn echo_server_script(port: u16) -> String { - format!( - r" -import socket, sys, signal -signal.signal(signal.SIGHUP, signal.SIG_IGN) -signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) -port = {port} -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -sock.bind(('127.0.0.1', port)) -sock.listen(1) -sock.settimeout(300) -print('echo-server-ready', flush=True) -try: - while True: - conn, _ = sock.accept() - data = conn.recv(4096) - if data: - conn.sendall(b'echo:' + data) - conn.close() -except (socket.timeout, OSError): - pass -finally: - sock.close() -" - ) -} - -/// Create a sandbox with a TCP echo server, forward the port locally, send -/// data through it, and verify the echoed response. -#[tokio::test] -async fn port_forward_echo() { - let port = find_free_port(); - let script = echo_server_script(port); - - // --------------------------------------------------------------- - // Step 1 — Create a sandbox with the echo server running. - // --------------------------------------------------------------- - let mut guard = SandboxGuard::create_keep(&["python3", "-c", &script], "echo-server-ready") - .await - .expect("sandbox create with echo server"); - - // --------------------------------------------------------------- - // Step 2 — Start port forwarding in the background. - // --------------------------------------------------------------- - let mut forward_child = guard.spawn_forward(port).expect("spawn port forward"); - - // Wait for the local port to accept connections. - wait_for_port("127.0.0.1", port, Duration::from_secs(30)) - .await - .expect("local port should open for forwarding"); - - // Give the SSH tunnel a moment to fully establish the direct-tcpip channel. - tokio::time::sleep(Duration::from_secs(2)).await; - - // --------------------------------------------------------------- - // Step 3 — Send data through the forwarded port and verify response. - // --------------------------------------------------------------- - let expected = "echo:hello-nav"; - let mut last_response = String::new(); - - for attempt in 1..=5 { - match try_echo(port).await { - Ok(resp) if resp.starts_with(expected) => { - last_response = resp; - break; - } - Ok(resp) => { - last_response = resp; - eprintln!("attempt {attempt}: unexpected response '{last_response}', retrying..."); - } - Err(e) => { - eprintln!("attempt {attempt}: connection error: {e}, retrying..."); - } - } - tokio::time::sleep(Duration::from_secs(2)).await; - } - - assert!( - last_response.starts_with(expected), - "expected response starting with '{expected}', got '{last_response}'" - ); - - // --------------------------------------------------------------- - // Cleanup — kill forward process, then sandbox guard handles the rest. - // --------------------------------------------------------------- - let _ = forward_child.kill().await; - let _ = forward_child.wait().await; - guard.cleanup().await; -} - -/// Attempt to send `hello-nav\n` to the echo server and read the response. -#[allow( - clippy::disallowed_methods, - reason = "the e2e workspace does not depend on openshell-core" -)] -async fn try_echo(port: u16) -> Result { - let mut stream = TcpStream::connect(format!("127.0.0.1:{port}")) - .await - .map_err(|e| format!("connect: {e}"))?; - - stream - .write_all(b"hello-nav\n") - .await - .map_err(|e| format!("write: {e}"))?; - - let mut buf = vec![0u8; 4096]; - let n = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buf)) - .await - .map_err(|_| "read timeout".to_string())? - .map_err(|e| format!("read: {e}"))?; - - let response = String::from_utf8_lossy(&buf[..n]) - .trim_end_matches(['\r', '\n']) - .to_string(); - - Ok(response) -} diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs deleted file mode 100644 index 09eda47615..0000000000 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ /dev/null @@ -1,1683 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for the transparent sandbox egress pipeline. -//! -//! Workloads connect directly to their requested destinations. Seccomp -//! notification diverts those sockets to the supervisor without proxy -//! environment variables or explicit CONNECT requests. These tests cover: -//! - live policy reloads affect new requests and close a pre-existing HTTP -//! stream before its next request is forwarded; -//! - `tls: skip` selects a byte-transparent TCP relay; -//! - provider placeholders in HTTP headers and opted-in REST bodies are -//! resolved without appearing in test output. - -use std::io::{self, Error, ErrorKind, Write}; -use std::process::Stdio; -use std::sync::{ - Arc, Mutex, - atomic::{AtomicUsize, Ordering}, -}; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::{SupportContainer, e2e_network_name}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; -use serial_test::serial; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const TEST_SERVER_HOST: &str = "host.openshell.internal"; -const PROVIDER_NAME: &str = "e2e-proxy-egress-credentials"; -const PROVIDER_PROFILE_ID: &str = "e2e-proxy-egress-credentials"; -const TOKEN_ENV: &str = "PROXY_E2E_TOKEN"; -const TEST_SECRET: &str = "sk-e2e-proxy-egress-secret"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -const PRIVATE_ALLOWED_IPS: &str = r#" allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7""#; -static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|error| format!("failed to spawn openshell {}: {error}", args.join(" ")))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn wait_for_sandbox_logs( - sandbox_name: &str, - expected: impl Fn(&str) -> bool, -) -> Result { - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); - - loop { - let logs = run_cli(&[ - "logs", - sandbox_name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) - .await?; - if expected(&logs) { - return Ok(logs); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "timed out waiting for expected sandbox logs:\n{logs}" - )); - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } -} - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["provider", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["profile", "delete", id]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn create_bound_provider(name: &str) -> Result { - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - run_cli(&[ - "provider", - "create", - "--name", - name, - "--type", - PROVIDER_PROFILE_ID, - "--credential", - &credential, - ]) - .await -} - -fn write_credential_profile(port: u16) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create provider profile: {error}"))?; - let profile = format!( - r#"id: {PROVIDER_PROFILE_ID} -display_name: E2E proxy egress credentials -category: other -credentials: - - name: proxy_e2e_token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_SERVER_HOST} - port: {port} - path: /probe - protocol: rest - access: full - enforcement: enforce - request_body_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" -binaries: ["/**"] -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write provider profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush provider profile: {error}"))?; - Ok(file) -} - -fn write_policy_document( - host: &str, - port: u16, - endpoint_options: &str, - network_middlewares: &str, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -{network_middlewares} -network_policies: - proxy_egress_test: - name: proxy_egress_test - endpoints: - - host: {host} - port: {port} -{endpoint_options} -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { - write_policy_document(host, port, endpoint_options, "") -} - -fn write_middleware_policy( - host: &str, - port: u16, - endpoint_options: &str, - on_error: &str, -) -> Result { - let network_middlewares = format!( - r#"network_middlewares: - regex-redactor: - name: Redact API tokens - middleware: openshell/regex - order: 10 - config: - mode: redact - on_error: {on_error} - endpoints: - include: ["{host}"] - exclude: [] -"# - ); - write_policy_document(host, port, endpoint_options, &network_middlewares) -} - -fn write_denied_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: {} -"#; - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_ambiguous_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - terminating: - name: terminating - endpoints: - - host: {host} - port: {port} -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" - passthrough: - name: passthrough - endpoints: - - host: {host} - port: {port} - tls: skip -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_destination_denial_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - destination_denials: - name: destination_denials - endpoints: - - { host: 169.254.169.254, port: 80 } - - { host: 127.0.0.1, port: 80 } - - { host: 203.0.113.10, port: 6443 } - - host: 203.0.113.10 - port: 8080 - allowed_ips: ["198.51.100.0/24"] - binaries: - - path: "/**" -"#; - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_ip_literal_success_policy( - explicit_ip: &str, - implicit_ip: &str, - port: u16, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - destination_successes: - name: destination_successes - endpoints: - - host: {ip} - port: {port} - allowed_ips: ["{explicit_ip}/32"] - - host: {implicit_ip} - port: {port} - binaries: - - path: "/**" -"#, - ip = explicit_ip, - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn policy_path(file: &NamedTempFile) -> String { - file.path() - .to_str() - .expect("temporary policy path should be utf-8") - .to_string() -} - -async fn read_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { - let mut data = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Ok(data); - } - data.extend_from_slice(&buffer[..read]); - if data.windows(marker.len()).any(|window| window == marker) { - return Ok(data); - } - } -} - -fn header_end(bytes: &[u8]) -> Option { - bytes - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|position| position + 4) -} - -fn content_length(headers: &[u8]) -> io::Result { - let text = - std::str::from_utf8(headers).map_err(|error| Error::new(ErrorKind::InvalidData, error))?; - Ok(text - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0)) -} - -async fn read_http_request(stream: &mut TcpStream) -> io::Result>> { - let mut request = read_until(stream, b"\r\n\r\n").await?; - if request.is_empty() { - return Ok(None); - } - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let body_length = content_length(&request[..headers_end])?; - let total_length = headers_end + body_length; - while request.len() < total_length { - let mut buffer = vec![0_u8; total_length - request.len()]; - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Err(Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP body")); - } - request.extend_from_slice(&buffer[..read]); - } - request.truncate(total_length); - Ok(Some(request)) -} - -struct KeepAliveHttpServer { - port: u16, - connections: Arc, - task: JoinHandle<()>, -} - -impl KeepAliveHttpServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind HTTP server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read HTTP server address: {error}"))? - .port(); - let connections = Arc::new(AtomicUsize::new(0)); - let task_connections = Arc::clone(&connections); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - task_connections.fetch_add(1, Ordering::AcqRel); - tokio::spawn(async move { - let _ = handle_keep_alive_connection(stream).await; - }); - } - }); - Ok(Self { - port, - connections, - task, - }) - } - - fn connection_count(&self) -> usize { - self.connections.load(Ordering::Acquire) - } -} - -impl Drop for KeepAliveHttpServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_keep_alive_connection(mut stream: TcpStream) -> io::Result<()> { - while let Some(request) = read_http_request(&mut stream).await? { - let close = String::from_utf8_lossy(&request) - .lines() - .any(|line| line.eq_ignore_ascii_case("connection: close")); - let connection = if close { "close" } else { "keep-alive" }; - let response = - format!("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\nok"); - stream.write_all(response.as_bytes()).await?; - if close { - return Ok(()); - } - } - Ok(()) -} - -struct EchoServer { - port: u16, - observed: Arc>>, - task: JoinHandle<()>, -} - -struct RequestBodyEchoServer { - port: u16, - task: JoinHandle<()>, -} - -impl RequestBodyEchoServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind request body echo server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read request body echo server address: {error}"))? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(async move { - let _ = handle_request_body_echo(stream).await; - }); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for RequestBodyEchoServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_request_body_echo(mut stream: TcpStream) -> io::Result<()> { - let request = read_http_request(&mut stream) - .await? - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let body = &request[headers_end..]; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - stream.write_all(response.as_bytes()).await?; - stream.write_all(body).await -} - -struct PipelineProbeServer { - port: u16, - observed: Arc>>, - task: JoinHandle<()>, -} - -impl PipelineProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind pipeline probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read pipeline probe address: {error}"))? - .port(); - let observed = Arc::new(Mutex::new(Vec::new())); - let observed_task = observed.clone(); - let task = tokio::spawn(async move { - while let Ok((mut stream, _)) = listener.accept().await { - let observed = observed_task.clone(); - tokio::spawn(async move { - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - match tokio::time::timeout( - std::time::Duration::from_millis(200), - stream.read(&mut buffer), - ) - .await - { - Ok(Ok(0)) | Err(_) => break, - Ok(Ok(read)) => request.extend_from_slice(&buffer[..read]), - Ok(Err(_)) => return, - } - } - observed.lock().unwrap().extend_from_slice(&request); - // Keep the first response reusable so the queued request reaches policy evaluation. - let _ = stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await; - }); - } - }); - Ok(Self { - port, - observed, - task, - }) - } - - fn observed_request(&self) -> Vec { - self.observed.lock().unwrap().clone() - } -} - -impl Drop for PipelineProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -impl EchoServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind echo server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read echo server address: {error}"))? - .port(); - let observed = Arc::new(Mutex::new(Vec::new())); - let task_observed = Arc::clone(&observed); - let task = tokio::spawn(async move { - while let Ok((mut stream, _)) = listener.accept().await { - let observed = Arc::clone(&task_observed); - tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - loop { - let Ok(read) = stream.read(&mut buffer).await else { - break; - }; - if read == 0 { - break; - } - observed.lock().unwrap().extend_from_slice(&buffer[..read]); - if stream.write_all(&buffer[..read]).await.is_err() { - break; - } - } - }); - } - }); - Ok(Self { - port, - observed, - task, - }) - } - - fn observed_bytes(&self) -> Vec { - self.observed.lock().unwrap().clone() - } -} - -impl Drop for EchoServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -struct CredentialProbeServer { - port: u16, - task: JoinHandle<()>, -} - -impl CredentialProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind credential probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read credential probe address: {error}"))? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(async move { - let _ = handle_credential_probe(stream).await; - }); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for CredentialProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { - let request = read_http_request(&mut stream) - .await? - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let headers = String::from_utf8_lossy(&request[..headers_end]); - let expected_authorization = format!("Bearer {TEST_SECRET}"); - let header_resolved = headers.lines().any(|line| { - line.split_once(':').is_some_and(|(name, value)| { - name.eq_ignore_ascii_case("authorization") && value.trim() == expected_authorization - }) - }); - let body_resolved = request[headers_end..] - .windows(TEST_SECRET.len()) - .any(|window| window == TEST_SECRET.as_bytes()); - let saw_placeholder = request - .windows(PLACEHOLDER_PREFIX.len()) - .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()); - let body = serde_json::json!({ - "body_resolved": body_resolved, - "header_resolved": header_resolved, - "saw_placeholder": saw_placeholder, - }) - .to_string(); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(response.as_bytes()).await -} - -fn transparent_status_script(host: &str, port: u16) -> String { - format!( - r#" -import json -import socket - -HOST = {host:?} -PORT = {port} - -def read_headers(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def status(response): - parts = response.split(None, 2) - return int(parts[1]) if len(parts) > 1 else 0 - -def request_status(path): - target = f"{{HOST}}:{{PORT}}" - try: - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall( - f"GET {{path}} HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - except OSError as error: - return {{"errno": error.errno, "error": repr(error)}} - -print(json.dumps({{ - "first": request_status("/first"), - "second": request_status("/second"), -}}, sort_keys=True)) -"#, - host = host, - port = port, - ) -} - -fn persistent_transparent_script(host: &str, port: u16) -> String { - format!( - r#" -import json -import os -import socket -import time - -HOST = {host:?} -PORT = {port} -READY = "/tmp/proxy-reload-ready" -GO = "/tmp/proxy-reload-go" -RESULT = "/tmp/proxy-reload-result" - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - return 0 - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - return 0 - body += chunk - return int(headers.split(None, 2)[1]) - -target = f"{{HOST}}:{{PORT}}" -failed_closed = False -second_status = 0 -try: - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall( - f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() - ) - if read_response(sock) != 200: - raise RuntimeError("initial tunneled request was denied") - open(READY, "w").close() - deadline = time.monotonic() + 120 - while not os.path.exists(GO) and time.monotonic() < deadline: - time.sleep(0.1) - if not os.path.exists(GO): - raise RuntimeError("timed out waiting for policy reload signal") - try: - sock.sendall( - f"GET /after-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - second_status = read_response(sock) - except OSError: - second_status = 0 - failed_closed = second_status != 200 -finally: - with open(RESULT, "w") as result: - json.dump({{"failed_closed": failed_closed, "second_status": second_status}}, result, sort_keys=True) -"#, - host = host, - port = port, - ) -} - -async fn wait_for_sandbox_file(guard: &SandboxGuard, path: &str, log_path: &str) -> String { - let script = format!( - r#"import os, time -deadline = time.monotonic() + 60 -while not os.path.exists({path:?}) and time.monotonic() < deadline: - time.sleep(0.1) -if not os.path.exists({path:?}): - if os.path.exists({log_path:?}): - print(open({log_path:?}).read()) - raise SystemExit("timed out waiting for {path}") -print(open({path:?}).read()) -"# - ); - guard - .exec(&["python3", "-c", &script]) - .await - .unwrap_or_else(|error| panic!("wait for sandbox file {path}: {error}")) -} - -fn parse_json_line(output: &str) -> Value { - output - .lines() - .filter_map(|line| serde_json::from_str::(line.trim()).ok()) - .next_back() - .unwrap_or_else(|| panic!("missing JSON result in sandbox output:\n{output}")) -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn policy_reload_updates_transparent_requests_and_closes_existing_http_stream() { - let server = KeepAliveHttpServer::start() - .await - .expect("start keep-alive HTTP server"); - let policy_a = write_policy(TEST_SERVER_HOST, server.port, "").expect("write policy A"); - let policy_b = write_denied_policy().expect("write policy B"); - let policy_a_path = policy_path(&policy_a); - let policy_b_path = policy_path(&policy_b); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_a_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_a_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("wait for policy A"); - - let persistent_script = persistent_transparent_script(TEST_SERVER_HOST, server.port); - guard - .exec(&[ - "sh", - "-c", - "nohup python3 -c \"$1\" >/tmp/proxy-reload-client.log 2>&1 &", - "proxy-reload-client", - &persistent_script, - ]) - .await - .expect("start persistent transparent client"); - wait_for_sandbox_file( - &guard, - "/tmp/proxy-reload-ready", - "/tmp/proxy-reload-client.log", - ) - .await; - - let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); - let before = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests before reload"); - let before = parse_json_line(&before); - assert_eq!( - before["first"], 200, - "first request before reload: {before}" - ); - assert_eq!( - before["second"], 200, - "second request before reload: {before}" - ); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("publish and wait for policy B"); - - guard - .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) - .await - .expect("release persistent transparent client"); - let stale_tunnel = wait_for_sandbox_file( - &guard, - "/tmp/proxy-reload-result", - "/tmp/proxy-reload-client.log", - ) - .await; - let stale_tunnel = parse_json_line(&stale_tunnel); - assert_eq!( - stale_tunnel["failed_closed"], true, - "existing transparent HTTP stream forwarded after policy reload: {stale_tunnel}" - ); - - let after = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests after reload"); - let after = parse_json_line(&after); - assert_ne!(after["first"], 200, "first request after reload: {after}"); - assert_ne!(after["second"], 200, "second request after reload: {after}"); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { - let server = KeepAliveHttpServer::start() - .await - .expect("start keep-alive HTTP server"); - let valid_policy = write_policy(TEST_SERVER_HOST, server.port, "").expect("write valid policy"); - let ambiguous_policy = - write_ambiguous_policy(TEST_SERVER_HOST, server.port).expect("write ambiguous policy"); - let valid_policy_path = policy_path(&valid_policy); - let ambiguous_policy_path = policy_path(&ambiguous_policy); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &valid_policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &valid_policy_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("wait for valid policy"); - - let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); - let before = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests before invalid update"); - let before = parse_json_line(&before); - assert_eq!( - before["first"], 200, - "first request before update: {before}" - ); - assert_eq!( - before["second"], 200, - "second request before update: {before}" - ); - let history_before = run_cli(&["policy", "list", &guard.name]) - .await - .expect("list policy history before rejected update"); - let connections_before_rejection = server.connection_count(); - - let update_error = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &ambiguous_policy_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect_err("ambiguous policy must be rejected before persistence"); - assert!( - update_error.contains("ambiguity validation failed"), - "policy update should explain the ambiguity:\n{update_error}" - ); - - let history_after = run_cli(&["policy", "list", &guard.name]) - .await - .expect("list policy history after rejected update"); - assert_eq!( - history_after, history_before, - "rejected policy must not create a revision" - ); - - let after_rejection = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests after rejected update"); - let after_rejection = parse_json_line(&after_rejection); - assert_eq!( - after_rejection["first"], 200, - "first request should keep using the active valid policy: {after_rejection}" - ); - assert_eq!( - after_rejection["second"], 200, - "second request should keep using the active valid policy: {after_rejection}" - ); - assert!( - server.connection_count() > connections_before_rejection, - "active-policy requests should still contact the upstream server" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn transparent_destination_denials_fail_connect_with_eacces() { - let policy = write_destination_denial_policy().expect("write destination denial policy"); - let policy_path = policy_path(&policy); - let script = r#" -import json -import socket - -targets = { - "metadata": ("169.254.169.254", 80), - "control_plane": ("203.0.113.10", 6443), - "outside_allowed_ips": ("203.0.113.10", 8080), -} -result = {} -for name, target in targets.items(): - try: - with socket.create_connection(target, timeout=10): - result[name] = 0 - except OSError as error: - result[name] = error.errno -print(json.dumps(result, sort_keys=True)) -"#; - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", script]) - .await - .expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for name in ["metadata", "control_plane", "outside_allowed_ips"] { - assert_eq!(result[name], 13, "{name} should fail with EACCES: {result}"); - } -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_transparently() { - if e2e_network_name().is_none() { - eprintln!("skipping IP-literal success assertions without a shared container network"); - return; - } - - const HTTP_SERVER: &str = r#" -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - body = b"ok" - self.send_response(200) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - pass - -ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - let explicit_server = - SupportContainer::start_python("explicit-ip.openshell.test", HTTP_SERVER, 8000) - .await - .expect("start explicit allowed_ips support container"); - let implicit_server = - SupportContainer::start_python("implicit-ip.openshell.test", HTTP_SERVER, 8000) - .await - .expect("start implicit IP-literal support container"); - let explicit_ip = explicit_server.ip().expect("explicit support container IP"); - let implicit_ip = implicit_server.ip().expect("implicit support container IP"); - let policy = write_ip_literal_success_policy(&explicit_ip, &implicit_ip, 8000) - .expect("write IP literal policy"); - let policy_path = policy_path(&policy); - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - for (mode, destination) in [ - ("explicit_allowed_ips", explicit_ip.as_str()), - ("implicit_ip_literal", implicit_ip.as_str()), - ] { - let output = guard - .exec(&[ - "python3", - "-c", - &transparent_status_script(destination, 8000), - ]) - .await - .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); - let statuses = parse_json_line(&output); - assert_eq!(statuses["first"], 200, "{mode} first request: {statuses}"); - assert_eq!(statuses["second"], 200, "{mode} second request: {statuses}"); - } - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_policy(TEST_SERVER_HOST, server.port, " tls: skip") - .expect("write tls: skip policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - echoed = b"" - while len(echoed) < len(PAYLOAD): - chunk = sock.recv(len(PAYLOAD) - len(echoed)) - if not chunk: - break - echoed += chunk - if echoed != PAYLOAD: - raise RuntimeError("opaque payload changed in transit") -print("RAW_RELAY_OK") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("RAW_RELAY_OK"), - "raw relay did not preserve the opaque payload:\n{}", - guard.create_output - ); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn middleware_redacts_transparent_request_bodies() { - let server = RequestBodyEchoServer::start() - .await - .expect("start request body echo server"); - let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") - .expect("write middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import json -import socket - -HOST = {host:?} -PORT = {port} -SECRET = "sk-1234567890abcdef" - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - raise RuntimeError("incomplete response headers") - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - status = int(headers.split(None, 2)[1]) - if status != 200: - raise RuntimeError(f"request failed with HTTP {{status}}: {{body!r}}") - return json.loads(body[:length]) - -def request_bytes(target): - body = json.dumps({{"api_key": SECRET}}, separators=(",", ":")).encode() - return ( - f"POST {{target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - -def request_once(): - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request_bytes("/middleware")) - return read_response(sock) - -print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for request in ["first", "second"] { - assert_eq!( - result[request]["api_key"], "[REDACTED]", - "{request} did not deliver the middleware-transformed body: {result}" - ); - } -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn fail_closed_middleware_blocks_uninspectable_transparent_payload_before_upstream() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") - .expect("write fail-closed middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - denial = b"" - while True: - try: - chunk = sock.recv(4096) - except ConnectionResetError: - break - if not chunk: - break - denial += chunk - if denial and ( - b"HTTP/1.1 403 Forbidden" not in denial - or b"unsupported_l7_protocol" not in denial - ): - raise RuntimeError(f"missing fail-closed middleware denial: {{denial!r}}") -print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - let output = guard - .exec(&["python3", "-c", &script]) - .await - .expect("exercise uninspectable fail-closed middleware"); - assert!( - output.contains("UNINSPECTABLE_MIDDLEWARE_BLOCKED"), - "uninspectable payload was not blocked:\n{output}" - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - assert!( - server.observed_bytes().is_empty(), - "uninspectable payload reached upstream before middleware denial" - ); - - let logs = wait_for_sandbox_logs(&guard.name, |logs| { - logs.contains("openshell.middleware.traffic_uninspectable") - && logs - .contains("Unsupported tunnel protocol cannot be inspected by required middleware") - }) - .await - .expect("fetch sandbox logs after middleware denial"); - assert!( - logs.contains("openshell.middleware.traffic_uninspectable") - && logs - .contains("Unsupported tunnel protocol cannot be inspected by required middleware"), - "OCSF logs should explain the fail-closed denial:\n{logs}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn fail_open_middleware_bypasses_uninspectable_transparent_tls_skip() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_middleware_policy( - TEST_SERVER_HOST, - server.port, - " tls: skip", - "fail_open", - ) - .expect("write fail-open middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - echoed = b"" - while len(echoed) < len(PAYLOAD): - chunk = sock.recv(len(PAYLOAD) - len(echoed)) - if not chunk: - break - echoed += chunk - if echoed != PAYLOAD: - raise RuntimeError(f"fail-open middleware did not preserve raw relay: {{echoed!r}}") -print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard - .create_output - .contains("UNINSPECTABLE_MIDDLEWARE_BYPASSED"), - "fail-open middleware did not bypass uninspectable traffic:\n{}", - guard.create_output - ); - assert_eq!( - server.observed_bytes(), - [0x00, 0xff, 0x13, 0x37, 0x80] - .into_iter() - .chain(*b"middleware-bypass") - .collect::>(), - "upstream did not receive the unchanged fail-open payload" - ); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn transparent_pipeline_never_reaches_upstream_as_first_request_overflow() { - let server = PipelineProbeServer::start() - .await - .expect("start pipeline probe server"); - let endpoint_options = r#" protocol: rest - enforcement: enforce - rules: - - allow: - method: GET - path: "/allowed""#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) - .expect("write pipeline policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -target = "{host}:{port}" -first = ( - f"GET /allowed HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" -) -second = ( - f"POST /blocked HTTP/1.1\r\n" - f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" -) -with socket.create_connection(({host:?}, {port}), timeout=10) as sock: - sock.sendall((first + second).encode()) - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk -responses = response.count(b"HTTP/1.1 ") -first_status = response.split(b"\r\n", 1)[0] -if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: - raise RuntimeError(f"unexpected pipelined response: {{response!r}}") -print("TRANSPARENT_PIPELINE_DENIED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("TRANSPARENT_PIPELINE_DENIED"), - "transparent HTTP stream did not deny the disallowed pipelined request:\n{}", - guard.create_output - ); - - let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); - assert!(observed.starts_with("GET /allowed HTTP/1.1\r\n")); - assert!( - !observed.to_ascii_lowercase().contains("\r\nconnection:"), - "shared relay must remove hop-by-hop connection headers:\n{observed}" - ); - assert!(!observed.contains("/blocked")); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn chunked_pipeline_is_authorized_separately_before_reaching_upstream() { - let server = PipelineProbeServer::start() - .await - .expect("start pipeline probe server"); - let endpoint_options = r#" protocol: rest - enforcement: enforce - rules: - - allow: - method: POST - path: "/allowed""#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) - .expect("write pipeline policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -target = "{host}:{port}" -first = ( - f"POST /allowed HTTP/1.1\r\n" - f"Host: {{target}}\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\n\r\n" - f"0\r\n\r\n" -) -second = ( - f"DELETE /blocked HTTP/1.1\r\n" - f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" -) -with socket.create_connection(({host:?}, {port}), timeout=10) as sock: - sock.sendall((first + second).encode()) - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk -responses = response.count(b"HTTP/1.1 ") -first_status = response.split(b"\r\n", 1)[0] -if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: - raise RuntimeError(f"unexpected chunked pipeline response: {{response!r}}") -print("CHUNKED_PIPELINE_DENIED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("CHUNKED_PIPELINE_DENIED"), - "transparent HTTP stream did not evaluate the chunked pipeline separately:\n{}", - guard.create_output - ); - - let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); - assert!(observed.starts_with("POST /allowed HTTP/1.1\r\n")); - assert!(observed.ends_with("0\r\n\r\n")); - assert!(!observed.contains("/blocked")); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn http_credentials_are_rewritten_in_transparent_headers_and_bodies() { - let _provider_lock = PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let result = async { - let server = CredentialProbeServer::start().await?; - let profile = write_credential_profile(server.port)?; - let profile_path = profile.path().to_string_lossy().into_owned(); - run_cli(&["profile", "import", "--file", &profile_path]).await?; - create_bound_provider(PROVIDER_NAME).await?; - let endpoint_options = r#" path: /probe - protocol: rest - enforcement: enforce - request_body_credential_rewrite: true - access: full"#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options)?; - let policy_path = policy_path(&policy); - let script = format!( - r#" -import json -import os -import socket - -HOST = {host:?} -PORT = {port} -TOKEN = os.environ[{token_env:?}] - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - raise RuntimeError("incomplete response headers") - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - code = int(headers.split(None, 2)[1]) - if code != 200: - raise RuntimeError(f"request failed with HTTP {{code}}") - return json.loads(body[:length]) - -def request_bytes(target): - body = json.dumps({{"credential": TOKEN}}, separators=(",", ":")).encode() - return ( - f"POST {{target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - f"Authorization: Bearer {{TOKEN}}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - -def request_once(): - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request_bytes("/probe")) - return read_response(sock) - -print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) -"#, - host = TEST_SERVER_HOST, - port = server.port, - token_env = TOKEN_ENV, - ); - - SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await - } - .await; - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let guard = result.expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for request in ["first", "second"] { - assert_eq!( - result[request]["header_resolved"], true, - "{request} header placeholder was not resolved: {result}" - ); - assert_eq!( - result[request]["body_resolved"], true, - "{request} body placeholder was not resolved: {result}" - ); - assert_eq!( - result[request]["saw_placeholder"], false, - "{request} leaked an unresolved placeholder upstream: {result}" - ); - } - assert!( - !guard.create_output.contains(TEST_SECRET), - "sandbox output exposed the raw provider credential:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(PLACEHOLDER_PREFIX), - "sandbox output exposed an unresolved provider placeholder:\n{}", - guard.create_output - ); -} diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs deleted file mode 100644 index 1150e020bc..0000000000 --- a/e2e/rust/tests/websocket_conformance.rs +++ /dev/null @@ -1,517 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E regression: WebSocket credential placeholders are resolved on the -//! sandbox path after an RFC 6455 upgrade. -//! -//! The sandbox process sends its provider-managed placeholder in a masked text -//! frame. The local upstream only reports whether it saw the real secret and -//! whether any placeholder survived; it never echoes payload bytes, placeholder -//! text, or secret material back into test output. - -use std::io::{self, Error, ErrorKind, Write}; -use std::process::Stdio; -use std::sync::Mutex; - -use base64::Engine as _; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::sandbox::SandboxGuard; -use sha1::{Digest, Sha1}; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -const PROVIDER_NAME: &str = "e2e-websocket-conformance"; -const PROVIDER_PROFILE_ID: &str = "e2e-websocket-conformance"; -const TEST_SERVER_HOST: &str = "host.openshell.internal"; -const TEST_SECRET: &str = "sk-e2e-websocket-conformance-secret"; -const TOKEN_ENV: &str = "WS_E2E_TOKEN"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|e| format!("failed to spawn openshell {}: {e}", args.join(" ")))?; - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["profile", "delete", id]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn create_bound_provider(name: &str) -> Result { - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - run_cli(&[ - "provider", - "create", - "--name", - name, - "--type", - PROVIDER_PROFILE_ID, - "--credential", - &credential, - ]) - .await -} - -fn write_credential_profile(port: u16) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create provider profile: {error}"))?; - let profile = format!( - r#"id: {PROVIDER_PROFILE_ID} -display_name: E2E WebSocket conformance credentials -category: other -credentials: - - name: websocket_token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_SERVER_HOST} - port: {port} - path: /ws - protocol: websocket - enforcement: enforce - access: read-write - websocket_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write provider profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush provider profile: {error}"))?; - Ok(file) -} - -struct WebSocketProbeServer { - port: u16, - task: JoinHandle<()>, -} - -impl WebSocketProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|e| format!("bind websocket probe server: {e}"))?; - let port = listener - .local_addr() - .map_err(|e| format!("read websocket probe server address: {e}"))? - .port(); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - tokio::spawn(async move { - let _ = handle_websocket_probe_connection(stream).await; - }); - } - }); - - Ok(Self { port, task }) - } -} - -impl Drop for WebSocketProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { - let mut data = Vec::new(); - let mut buf = [0_u8; 1024]; - loop { - let read = stream.read(&mut buf).await?; - if read == 0 { - return Ok(data); - } - data.extend_from_slice(&buf[..read]); - if data.windows(marker.len()).any(|window| window == marker) { - return Ok(data); - } - } -} - -async fn read_websocket_text(stream: &mut TcpStream) -> io::Result { - let mut header = [0_u8; 2]; - stream.read_exact(&mut header).await?; - let length = match header[1] & 0x7F { - len @ 0..=125 => usize::from(len), - 126 => { - let mut bytes = [0_u8; 2]; - stream.read_exact(&mut bytes).await?; - usize::from(u16::from_be_bytes(bytes)) - } - 127 => { - let mut bytes = [0_u8; 8]; - stream.read_exact(&mut bytes).await?; - usize::try_from(u64::from_be_bytes(bytes)) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - } - _ => unreachable!(), - }; - - let mut mask = [0_u8; 4]; - if header[1] & 0x80 != 0 { - stream.read_exact(&mut mask).await?; - } else { - mask = [0, 0, 0, 0]; - } - - let mut payload = vec![0_u8; length]; - stream.read_exact(&mut payload).await?; - if header[1] & 0x80 != 0 { - for (index, byte) in payload.iter_mut().enumerate() { - *byte ^= mask[index % mask.len()]; - } - } - - String::from_utf8(payload).map_err(|e| { - Error::new( - ErrorKind::InvalidData, - format!("invalid websocket text: {e}"), - ) - }) -} - -async fn send_websocket_text(stream: &mut TcpStream, payload: &str) -> io::Result<()> { - let data = payload.as_bytes(); - let mut frame = Vec::with_capacity(data.len() + 10); - frame.push(0x81); - if data.len() < 126 { - frame.push( - u8::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))?, - ); - } else if data.len() <= usize::from(u16::MAX) { - frame.push(126); - frame.extend_from_slice( - &u16::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - .to_be_bytes(), - ); - } else { - frame.push(127); - frame.extend_from_slice( - &u64::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - .to_be_bytes(), - ); - } - frame.extend_from_slice(data); - stream.write_all(&frame).await -} - -fn header_value(request: &str, name: &str) -> Option { - request.lines().find_map(|line| { - let (header, value) = line.split_once(':')?; - if header.trim().eq_ignore_ascii_case(name) { - Some(value.trim().to_string()) - } else { - None - } - }) -} - -fn websocket_accept_for_key(key: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) -} - -async fn handle_websocket_probe_connection(mut stream: TcpStream) -> io::Result<()> { - let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; - let request = String::from_utf8_lossy(&request_bytes); - if !request.to_ascii_lowercase().contains("upgrade: websocket") { - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") - .await?; - return Ok(()); - } - - let accept = header_value(&request, "Sec-WebSocket-Key") - .map(|key| websocket_accept_for_key(&key)) - .ok_or_else(|| Error::new(ErrorKind::InvalidData, "missing Sec-WebSocket-Key"))?; - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Connection: Upgrade\r\n\ - Sec-WebSocket-Accept: {accept}\r\n\ - \r\n" - ); - stream.write_all(response.as_bytes()).await?; - - let text = read_websocket_text(&mut stream).await?; - let response = format!( - r#"{{"saw_placeholder": {}, "saw_secret": {}}}"#, - text.contains(PLACEHOLDER_PREFIX), - text.contains(TEST_SECRET) - ); - send_websocket_text(&mut stream, &response).await -} - -fn write_websocket_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - websocket_conformance: - name: websocket_conformance - endpoints: - - host: {host} - port: {port} - protocol: websocket - enforcement: enforce - access: read-write - websocket_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn websocket_client_script(host: &str, port: u16) -> String { - format!( - r#" -import base64 -import json -import os -import socket -import struct -import time - -HOST = {host:?} -PORT = {port} -TOKEN_ENV = {token_env:?} - -def recv_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_exact(sock, size): - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - raise EOFError("unexpected end of websocket frame") - data += chunk - return data - -def masked_text_frame(payload): - data = payload.encode("utf-8") - mask = os.urandom(4) - if len(data) < 126: - header = bytes([0x81, 0x80 | len(data)]) - elif len(data) <= 0xFFFF: - header = bytes([0x81, 0x80 | 126]) + struct.pack("!H", len(data)) - else: - header = bytes([0x81, 0x80 | 127]) + struct.pack("!Q", len(data)) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(data)) - return header + mask + masked - -def read_frame(sock): - first, second = read_exact(sock, 2) - length = second & 0x7F - if length == 126: - length = struct.unpack("!H", read_exact(sock, 2))[0] - elif length == 127: - length = struct.unpack("!Q", read_exact(sock, 8))[0] - mask = read_exact(sock, 4) if second & 0x80 else b"" - payload = read_exact(sock, length) - if mask: - payload = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) - return first, payload - -def transparent_socket_with_retry(host, port, timeout_seconds=20): - deadline = time.monotonic() + timeout_seconds - last_error = None - while time.monotonic() < deadline: - sock = None - try: - sock = socket.create_connection((host, port), timeout=5) - return sock - except OSError as error: - if sock is not None: - sock.close() - last_error = error - time.sleep(0.25) - raise last_error - -token = os.environ[TOKEN_ENV] -payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -key = base64.b64encode(os.urandom(16)).decode("ascii") -with transparent_socket_with_retry(HOST, PORT) as sock: - request = ( - "GET /ws HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError(f"websocket upgrade failed: {{response!r}}") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - result = json.loads(response_payload.decode("utf-8")) -results = {{"transparent": result}} -print(json.dumps(results, sort_keys=True)) -"#, - host = host, - port = port, - token_env = TOKEN_ENV, - ) -} - -#[tokio::test] -async fn websocket_text_placeholder_is_rewritten_transparently() { - let _provider_lock = PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let result = async { - let server = WebSocketProbeServer::start().await?; - let profile = write_credential_profile(server.port)?; - let profile_path = profile.path().to_string_lossy().into_owned(); - run_cli(&["profile", "import", "--file", &profile_path]).await?; - create_bound_provider(PROVIDER_NAME).await?; - let policy = write_websocket_policy(TEST_SERVER_HOST, server.port)?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| "temp policy path should be utf-8".to_string())? - .to_string(); - let script = websocket_client_script(TEST_SERVER_HOST, server.port); - - SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await - } - .await; - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let guard = result.expect("sandbox create"); - assert!( - guard - .create_output - .contains(r#""transparent": {"saw_placeholder": false, "saw_secret": true}"#), - "expected upstream to see only the resolved secret marker:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(TEST_SECRET), - "test output should not expose the raw WebSocket credential:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(PLACEHOLDER_PREFIX), - "test output should not expose unresolved credential placeholders:\n{}", - guard.create_output - ); -} diff --git a/tasks/test.toml b/tasks/test.toml index b2e599241c..ddcbec361c 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -208,7 +208,7 @@ run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tes ["e2e:rust:push"] description = "Run conformance and policy e2e tests against a Docker gateway that pushes configuration" -env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push live_policy_update policy_activation" } +env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push policy_activation" } run = "e2e/rust/e2e-docker.sh" ["e2e:workload:build"] @@ -231,13 +231,6 @@ exec e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml \ --features e2e-docker --test supervisor_middleware_http_hooks -- --nocapture """ -["e2e:websocket-conformance"] -description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" -run = [ - "CONTAINER_ENGINE=docker bash tasks/scripts/e2e-build-workload.sh", - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance", -] - ["e2e:mcp"] description = "Run MCP conformance e2e scenarios against one Docker-backed gateway (static defaults for spec 2025-11-25; set OPENSHELL_MCP_CONFORMANCE_SCENARIOS for a focused subset)" run = "bash e2e/mcp-conformance.sh" diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index 764889a1ee..d8b743b803 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -102,10 +102,6 @@ let "provider_auto_create" # Needs the SPIFFE fixtures that the wrapper sets up. Wrapper lane. "provider_token_exchange" - # One test times out reaching support-container IPs from the sandbox in the - # rootless guest; the other tests in the binary pass. Nextest archive - # filters cannot select individual tests, so keep the complete binary here. - "proxy_egress_pipeline" # Holds only the on-failure restart test until #5507; the rest moved to # testkit. Nextest archive filters cannot select individual tests, so keep # the complete binary in the follow-up bucket. From e43de85facfdb32c64e1b93e9bf1df35bd34c681 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:45:44 -0800 Subject: [PATCH 24/63] docs(testing): describe the network, policy update, and port forward suites Signed-off-by: Matthew Grossman --- TESTING.md | 8 ++++++++ tests/suites/conformance/README.md | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index 0e251d7ade..b9ae46db6f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -655,6 +655,14 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to the node's address. +The `routable-host-ip` feature suite (`tests/suites/features/routable-host-ip`) +checks that a policy endpoint whose host is an IP literal reaches that address, +with or without `allowed_ips`. The sandbox uses the address +`host.openshell.internal` resolves to, which must be one the supervisor lets an +IP literal reach. Rootless Podman's is link-local and always blocked, so CI runs +the suite on Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` +installers. + The `connect-ssh-supervision` feature suite (`tests/suites/features/connect-ssh-supervision`) checks how `sandbox connect` supervises its `ssh` process: it reconnects after losing the SSH transport and diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 84bb504247..adf367faff 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -20,7 +20,12 @@ target (`harness = false`), so `openshell-conformance run`, | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `create-upload` | `sandbox create --upload`: directory basename and nested files, multiple uploads, a single file | -| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST header and body placeholders and rewriting, WebSocket text and binary frames, live policy binding, static credentials bound to their profile endpoints | +| `network` | Direct connections that bypass the proxy are denied; reserved destinations fail with `EACCES`; localhost works without proxy settings; closed listeners release their ports | +| `network-l7` | REST, GraphQL, JSON-RPC, and MCP rules allow and deny requests; REST upgrades are refused | +| `network-egress` | Transparent egress: policy reloads close open connections, ambiguous updates are rejected, `tls: skip` relays raw bytes, middleware redacts or blocks traffic, pipelined requests are authorized one at a time | +| `policy-update` | `policy set` and `policy update`: revisions and versions, repeated updates, scoped endpoint changes, the revision for a sparse initial policy | +| `port-forward` | `forward start` reaches a server on the sandbox's loopback; `forward stop` closes the port | | `image` | A sandbox runs the default image or the one it is created from | | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | @@ -35,8 +40,9 @@ tmachine environment on CI. The other suites are ports of the old (credited in each file). `file-transfer` runs `git` on the machine that runs the tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine -environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts -servers (`tests/support/host-server`) on the machine that runs the tests and +environments). `credential-gating`, `network`, `network-l7`, and +`network-egress` start servers (`tests/support/host-server`) on the machine +that runs the tests and reaches them from sandboxes created from `testkit::python_image()` through `host.openshell.internal`. `ssh` runs `ssh` and `ssh-keygen` there and needs a named gateway, which `sandbox ssh-config` writes into its `ProxyCommand`. From 8ebd4f754b4999f54bbc5a4a28f54aacebc36301 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:09:45 -0800 Subject: [PATCH 25/63] test(conformance): keep sandbox names within the gateway's 19 character limit The gateway rejects sandbox names longer than 19 characters, which leaves 5 for a suffix after test--. The static credential binding test also needs the Python workload image, which has the sandbox user that the base policy runs as. Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/credential_gating.rs | 6 +++--- tests/suites/conformance/src/network.rs | 8 ++++---- tests/suites/conformance/src/network_egress.rs | 14 +++++++------- tests/suites/conformance/src/network_l7.rs | 11 +++++------ tests/suites/conformance/src/network_l7/mcp.rs | 2 +- tests/suites/conformance/src/policy_update.rs | 6 +++--- .../routable-host-ip/tests/routable_host_ip.rs | 2 +- tests/testkit/testkit/src/context.rs | 3 +++ 8 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 308fb49bd4..f6ff4a0c9d 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -234,7 +234,7 @@ async fn profile_rest_endpoint_rewrites_header_and_body(cx: &mut Context) -> tes let server = HttpRecorder::start("ok").await?; let sandbox = profile_endpoint_sandbox( cx, - "rewrite", + "rwrt", &format!( "{{host: {TEST_HOST}, port: {}, path: /probe, protocol: rest, access: full, \ enforcement: enforce, request_body_credential_rewrite: true}}", @@ -273,7 +273,7 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te let server = WebSocketRecorder::start().await?; let sandbox = profile_endpoint_sandbox( cx, - "wstext", + "wstxt", &format!( "{{host: {TEST_HOST}, port: {}, path: /ws, protocol: websocket, \ enforcement: enforce, access: read-write, websocket_credential_rewrite: true}}", @@ -339,9 +339,9 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: endpoint(server_a.port), endpoint(server_b.port), ))?; - // The default image, like the e2e test this replaces, for its bash. let sandbox = cx .sandbox("bind") + .image(testkit::python_image()) .policy(policy) .spec(|spec| spec.providers = vec![provider_a, provider_b]) .command(["sleep", "infinity"]) diff --git a/tests/suites/conformance/src/network.rs b/tests/suites/conformance/src/network.rs index b618509fd6..fdbc8c544f 100644 --- a/tests/suites/conformance/src/network.rs +++ b/tests/suites/conformance/src/network.rs @@ -28,7 +28,7 @@ const SOCKET_NOT_INHERITED: &str = include_str!("scripts/socket_not_inherited.py /// permission error well before the socket's 10-second timeout. #[testkit::test] async fn direct_connect_bypass_is_denied_promptly(cx: &mut Context) -> testkit::Result { - let sandbox = python_sandbox(cx, "bypass", policy::base()).await?; + let sandbox = python_sandbox(cx, "bypas", policy::base()).await?; let output = run_python(&sandbox, BYPASS_CLIENT, &[]).await?; let result = json_result(&output)?; testkit::ensure!( @@ -65,7 +65,7 @@ async fn reserved_destinations_fail_with_eacces_even_when_allowed( "#, binaries = python_binaries_yaml(), ))?; - let sandbox = python_sandbox(cx, "denials", policy).await?; + let sandbox = python_sandbox(cx, "deny", policy).await?; let output = run_python(&sandbox, DESTINATION_DENIALS_CLIENT, &[]).await?; expect_values( &json_result(&output)?, @@ -97,7 +97,7 @@ async fn localhost_is_reachable_without_proxy_environment(cx: &mut Context) -> t /// doesn't inherit the exited child's listener. #[testkit::test] async fn closed_or_exited_listener_releases_its_port(cx: &mut Context) -> testkit::Result { - let sandbox = python_sandbox(cx, "listen", policy::base()).await?; + let sandbox = python_sandbox(cx, "lsten", policy::base()).await?; let output = run_python(&sandbox, LISTENER_LIFETIME_PROBE, &[]).await?; let inode = json_result(&output)?[1].to_string(); run_python(&sandbox, SOCKET_NOT_INHERITED, &[&inode]) @@ -111,7 +111,7 @@ async fn closed_or_exited_listener_releases_its_port(cx: &mut Context) -> testki #[testkit::test] async fn exec_does_not_inherit_main_process_socket(cx: &mut Context) -> testkit::Result { let sandbox = cx - .sandbox("preconn") + .sandbox("prcon") .policy(policy::base()) .image(testkit::python_image()) .command(["python3", "-c", PRECONNECT_OWNER]) diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index f29d8fa63a..899e146eee 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -47,7 +47,7 @@ const PIPELINE_REPLY: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"; async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; let allowed = allow_server(server.port); - let sandbox = python_sandbox(cx, "reload", allowed.clone()).await?; + let sandbox = python_sandbox(cx, "reld", allowed.clone()).await?; policy::set(&sandbox, &allowed).await?; let port = server.port.to_string(); @@ -160,7 +160,7 @@ async fn tls_skip_relays_opaque_bytes(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let mut policy = policy::base(); allow(&mut policy, server.port).tls_skip(); - let sandbox = python_sandbox(cx, "tlsskip", policy).await?; + let sandbox = python_sandbox(cx, "tlskp", policy).await?; let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]; payload.extend_from_slice(b"not-http-or-tls"); payload.extend(0..64_u8); @@ -185,7 +185,7 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result }) .await?; let policy = middleware_policy(server.port, "fail_closed", "")?; - let sandbox = python_sandbox(cx, "redact", policy).await?; + let sandbox = python_sandbox(cx, "redct", policy).await?; let output = run_python( &sandbox, REDACTION_CLIENT, @@ -206,7 +206,7 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_closed", "")?; - let sandbox = python_sandbox(cx, "failclosed", policy).await?; + let sandbox = python_sandbox(cx, "fclos", policy).await?; run_python( &sandbox, UNINSPECTABLE_CLIENT, @@ -239,7 +239,7 @@ async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) - async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_open", " tls: skip\n")?; - let sandbox = python_sandbox(cx, "failopen", policy).await?; + let sandbox = python_sandbox(cx, "fopen", policy).await?; let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80]; payload.extend_from_slice(b"middleware-bypass"); run_python( @@ -264,7 +264,7 @@ async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> #[testkit::test] async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; - let sandbox = python_sandbox(cx, "pipeline", rest_policy(server.port, "GET")?).await?; + let sandbox = python_sandbox(cx, "pipe", rest_policy(server.port, "GET")?).await?; let target = format!("{TEST_HOST}:{}", server.port); let requests = format!( "GET /allowed HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n\ @@ -287,7 +287,7 @@ async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit #[testkit::test] async fn chunked_pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; - let sandbox = python_sandbox(cx, "chunked", rest_policy(server.port, "POST")?).await?; + let sandbox = python_sandbox(cx, "chunk", rest_policy(server.port, "POST")?).await?; let target = format!("{TEST_HOST}:{}", server.port); let requests = format!( "POST /allowed HTTP/1.1\r\nHost: {target}\r\nTransfer-Encoding: chunked\r\n\ diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs index e443eaf151..bbf71fde51 100644 --- a/tests/suites/conformance/src/network_l7.rs +++ b/tests/suites/conformance/src/network_l7.rs @@ -55,9 +55,8 @@ async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result #[testkit::test] async fn rest_upgrade_is_refused_in_enforce_and_audit_modes(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(rest_upstream).await?; - for enforcement in ["enforce", "audit"] { - let sandbox = - python_sandbox(cx, enforcement, rest_policy(server.port, enforcement)?).await?; + for (suffix, enforcement) in [("enfrc", "enforce"), ("audit", "audit")] { + let sandbox = python_sandbox(cx, suffix, rest_policy(server.port, enforcement)?).await?; run_python(&sandbox, REST_UPGRADE_CLIENT, &[TEST_HOST, &port(&server)]) .await .context(format!("{enforcement} mode"))? @@ -109,7 +108,7 @@ async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::R port = server.port, binaries = python_binaries_yaml(), ))?; - let sandbox = python_sandbox(cx, "graphql", policy).await?; + let sandbox = python_sandbox(cx, "gql", policy).await?; let output = run_python(&sandbox, GRAPHQL_CLIENT, &[TEST_HOST, &port(&server)]).await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { @@ -162,7 +161,7 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu port = server.port, binaries = python_binaries_yaml(), ))?; - let sandbox = python_sandbox(cx, "jsonrpc", policy).await?; + let sandbox = python_sandbox(cx, "jrpc", policy).await?; let output = run_python(&sandbox, JSONRPC_CLIENT, &[TEST_HOST, &port(&server)]).await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { @@ -212,7 +211,7 @@ async fn jsonrpc_default_audit_mode_still_denies_response_frames( port = server.port, binaries = python_binaries_yaml(), ))?; - let sandbox = python_sandbox(cx, "jsonaudit", policy).await?; + let sandbox = python_sandbox(cx, "jaud", policy).await?; let output = run_python(&sandbox, JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]).await?; expect_values( &json_result(&output)?, diff --git a/tests/suites/conformance/src/network_l7/mcp.rs b/tests/suites/conformance/src/network_l7/mcp.rs index 3ce7742b1e..c553516bb9 100644 --- a/tests/suites/conformance/src/network_l7/mcp.rs +++ b/tests/suites/conformance/src/network_l7/mcp.rs @@ -31,7 +31,7 @@ async fn mcp_sessionless_discovery_tools_and_subscription(cx: &mut Context) -> t /// Each legacy revision: initialization, then an allowed and a denied tool. #[testkit::test] async fn mcp_legacy_revisions_authorize_tools(cx: &mut Context) -> testkit::Result { - for (suffix, version) in [("mcp0326", "2025-03-26"), ("mcp0618", "2025-06-18")] { + for (suffix, version) in [("m0326", "2025-03-26"), ("m0618", "2025-06-18")] { run_profile(cx, suffix, &[version]).await?; } Ok(()) diff --git a/tests/suites/conformance/src/policy_update.rs b/tests/suites/conformance/src/policy_update.rs index a55efa1d58..4fe50706fd 100644 --- a/tests/suites/conformance/src/policy_update.rs +++ b/tests/suites/conformance/src/policy_update.rs @@ -34,7 +34,7 @@ const LOAD_TIMEOUT: Duration = Duration::from_mins(2); #[testkit::test] async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::Result { let sandbox = cx - .sandbox("polset") + .sandbox("pset") .policy(policy::base()) .command(["sleep", "infinity"]) .create_ready() @@ -97,7 +97,7 @@ async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::R #[testkit::test] async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> testkit::Result { let sandbox = cx - .sandbox("polscope") + .sandbox("pscop") .policy(scoped_policy()?) .command(["sleep", "infinity"]) .create_ready() @@ -209,7 +209,7 @@ async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> t #[testkit::test] async fn sparse_initial_policy_revision_is_loaded(cx: &mut Context) -> testkit::Result { let sandbox = cx - .cli_sandbox("sparse") + .cli_sandbox("spars") .policy_yaml(SPARSE_POLICY) .command(["sleep", "infinity"]) .create_ready() diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs index 613fe4d0a2..be414b0f0a 100644 --- a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -30,7 +30,7 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul let explicit = HttpRecorder::start("explicit").await?; let implicit = HttpRecorder::start("implicit").await?; let sandbox = cx - .sandbox("hostip") + .sandbox("hstip") .policy(policy::base()) .image(testkit::python_image()) .command(["sleep", "infinity"]) diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index d2be88374c..c7ab8839a1 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -172,6 +172,9 @@ impl Context { } /// A resource name unique to this run: `test--`. + /// + /// The gateway limits sandbox names to 19 characters, so a sandbox + /// suffix can have at most 5. pub fn name(&self, suffix: &str) -> String { format!("test-{}-{suffix}", self.run_id) } From 9c2f3d87c6d921f985e3c11dac5c21a8019e31c8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:32:06 -0800 Subject: [PATCH 26/63] test(features): use the shared http probe in routable-host-ip Drops the include of a conformance client script from another workspace. Signed-off-by: Matthew Grossman --- .../conformance/src/scripts/bypass_client.py | 26 ------ .../src/scripts/credential_rewrite_client.py | 46 ---------- .../src/scripts/destination_denials_client.py | 21 ----- .../conformance/src/scripts/echo_client.py | 23 ----- .../src/scripts/pipeline_client.py | 28 ------ .../src/scripts/redaction_client.py | 53 ----------- .../conformance/src/scripts/rest_client.py | 25 ------ .../src/scripts/rest_upgrade_client.py | 28 ------ .../conformance/src/scripts/status_client.py | 44 ---------- .../src/scripts/uninspectable_client.py | 30 ------- tests/suites/conformance/src/support.rs | 87 ------------------- tests/suites/features/Cargo.lock | 12 ++- .../features/routable-host-ip/Cargo.toml | 2 +- .../tests/routable_host_ip.rs | 55 ++++-------- .../scripts/bound_credential_client.sh | 0 .../sandbox-clients}/scripts/echo_server.py | 0 .../scripts/graphql_client.py | 0 .../scripts/jsonrpc_audit_client.py | 0 .../scripts/jsonrpc_client.py | 0 .../scripts/listener_lifetime_probe.py | 0 .../scripts/localhost_client.py | 0 .../sandbox-clients}/scripts/mcp_client.py | 0 .../scripts/preconnect_owner.py | 0 .../scripts/reload_keep_alive_client.py | 0 .../scripts/socket_not_inherited.py | 0 .../scripts/websocket_text_client.py | 0 26 files changed, 28 insertions(+), 452 deletions(-) delete mode 100644 tests/suites/conformance/src/scripts/bypass_client.py delete mode 100644 tests/suites/conformance/src/scripts/credential_rewrite_client.py delete mode 100644 tests/suites/conformance/src/scripts/destination_denials_client.py delete mode 100644 tests/suites/conformance/src/scripts/echo_client.py delete mode 100644 tests/suites/conformance/src/scripts/pipeline_client.py delete mode 100644 tests/suites/conformance/src/scripts/redaction_client.py delete mode 100644 tests/suites/conformance/src/scripts/rest_client.py delete mode 100644 tests/suites/conformance/src/scripts/rest_upgrade_client.py delete mode 100644 tests/suites/conformance/src/scripts/status_client.py delete mode 100644 tests/suites/conformance/src/scripts/uninspectable_client.py delete mode 100644 tests/suites/conformance/src/support.rs rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/bound_credential_client.sh (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/echo_server.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/graphql_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/jsonrpc_audit_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/jsonrpc_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/listener_lifetime_probe.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/localhost_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/mcp_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/preconnect_owner.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/reload_keep_alive_client.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/socket_not_inherited.py (100%) rename tests/{suites/conformance/src => support/sandbox-clients}/scripts/websocket_text_client.py (100%) diff --git a/tests/suites/conformance/src/scripts/bypass_client.py b/tests/suites/conformance/src/scripts/bypass_client.py deleted file mode 100644 index e3c1a75d51..0000000000 --- a/tests/suites/conformance/src/scripts/bypass_client.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Connects straight to an unroutable address, bypassing any proxy, and -# prints the outcome and how long it took. -import json, socket, time - -start = time.monotonic() -result = "unknown" -try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(10) - s.connect(("198.51.100.1", 80)) - result = "connected" - s.close() -except ConnectionRefusedError: - result = "refused" -except PermissionError: - result = "denied" -except socket.timeout: - result = "timeout" -except OSError as e: - result = f"error:{e}" - -elapsed_ms = int((time.monotonic() - start) * 1000) -print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=True) diff --git a/tests/suites/conformance/src/scripts/credential_rewrite_client.py b/tests/suites/conformance/src/scripts/credential_rewrite_client.py deleted file mode 100644 index b3d77a3ac0..0000000000 --- a/tests/suites/conformance/src/scripts/credential_rewrite_client.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# POSTs the provider's token from E2E_GATING_TOKEN in an Authorization header -# and a JSON body to argv[1]:argv[2]/probe twice, each on its own -# connection, and fails unless both get 200. Never prints the token. -import json -import os -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) -TOKEN = os.environ["E2E_GATING_TOKEN"] - - -def status(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - parts = data.split(None, 2) - return int(parts[1]) if len(parts) > 1 else 0 - - -def request_once(): - body = json.dumps({"credential": TOKEN}, separators=(",", ":")).encode() - request = ( - "POST /probe HTTP/1.1\r\n" - f"Host: {HOST}:{PORT}\r\n" - f"Authorization: Bearer {TOKEN}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {len(body)}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request) - return status(sock) - - -statuses = [request_once(), request_once()] -if statuses != [200, 200]: - raise SystemExit(f"expected two 200 responses; got {statuses}") -print("REWRITE_OK") diff --git a/tests/suites/conformance/src/scripts/destination_denials_client.py b/tests/suites/conformance/src/scripts/destination_denials_client.py deleted file mode 100644 index a4ddd6475e..0000000000 --- a/tests/suites/conformance/src/scripts/destination_denials_client.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Connects to addresses the policy names but the sandbox must still refuse, -# and prints each connect's errno as JSON, 0 for a connection. -import json -import socket - -targets = { - "metadata": ("169.254.169.254", 80), - "control_plane": ("203.0.113.10", 6443), - "outside_allowed_ips": ("203.0.113.10", 8080), -} -result = {} -for name, target in targets.items(): - try: - with socket.create_connection(target, timeout=10): - result[name] = 0 - except OSError as error: - result[name] = error.errno -print(json.dumps(result, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/echo_client.py b/tests/suites/conformance/src/scripts/echo_client.py deleted file mode 100644 index aea3b9da97..0000000000 --- a/tests/suites/conformance/src/scripts/echo_client.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Sends the bytes hex-encoded in argv[3] to the echo server at -# argv[1]:argv[2] and fails unless it reads the same bytes back. -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) -PAYLOAD = bytes.fromhex(sys.argv[3]) - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - echoed = b"" - while len(echoed) < len(PAYLOAD): - chunk = sock.recv(len(PAYLOAD) - len(echoed)) - if not chunk: - break - echoed += chunk -if echoed != PAYLOAD: - raise SystemExit(f"the payload changed in transit: {echoed!r}") -print("ECHO_OK") diff --git a/tests/suites/conformance/src/scripts/pipeline_client.py b/tests/suites/conformance/src/scripts/pipeline_client.py deleted file mode 100644 index 2ed5521bd9..0000000000 --- a/tests/suites/conformance/src/scripts/pipeline_client.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Sends the pipelined HTTP requests in argv[3] to argv[1]:argv[2] in one -# write, reads until the connection closes, and prints the status lines of -# the responses. -import json -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) -REQUESTS = sys.argv[3].encode() - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(REQUESTS) - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk -statuses = [ - line.decode(errors="replace") - for line in response.split(b"\r\n") - if line.startswith(b"HTTP/1.1 ") -] -print(json.dumps({"statuses": statuses})) diff --git a/tests/suites/conformance/src/scripts/redaction_client.py b/tests/suites/conformance/src/scripts/redaction_client.py deleted file mode 100644 index 6bb7bc9b99..0000000000 --- a/tests/suites/conformance/src/scripts/redaction_client.py +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# POSTs a JSON body holding an API key to the body-echo server at -# argv[1]:argv[2] twice, each on its own connection, and prints the bodies -# the server echoed. -import json -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) -SECRET = "sk-1234567890abcdef" - - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - raise RuntimeError("incomplete response headers") - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - status = int(headers.split(None, 2)[1]) - if status != 200: - raise RuntimeError(f"request failed with HTTP {status}: {body!r}") - return json.loads(body[:length]) - - -def request_once(): - body = json.dumps({"api_key": SECRET}, separators=(",", ":")).encode() - request = ( - "POST /middleware HTTP/1.1\r\n" - f"Host: {HOST}:{PORT}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {len(body)}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request) - return read_response(sock) - - -print(json.dumps({"first": request_once(), "second": request_once()}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/rest_client.py b/tests/suites/conformance/src/scripts/rest_client.py deleted file mode 100644 index d59543a760..0000000000 --- a/tests/suites/conformance/src/scripts/rest_client.py +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Prints the status of GET and POST to /allowed as JSON. -import json -import sys -import urllib.error -import urllib.request - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) - - -def status(method, data=None): - request = urllib.request.Request(f"http://{HOST}:{PORT}/allowed", data=data, method=method) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - - -print(json.dumps({"get": status("GET"), "post": status("POST", b"test")}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/rest_upgrade_client.py b/tests/suites/conformance/src/scripts/rest_upgrade_client.py deleted file mode 100644 index 305742fe46..0000000000 --- a/tests/suites/conformance/src/scripts/rest_upgrade_client.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Asks to upgrade a REST endpoint to WebSocket with an origin-form and an -# absolute-form target, and fails unless the sandbox refuses both. -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) - -for target in ["/allowed", f"http://{HOST}:{PORT}/allowed"]: - s = socket.create_connection((HOST, PORT), timeout=15) - request = ("GET " + target + f" HTTP/1.1\r\nHost: {HOST}:{PORT}\r\n" - "Connection: Upgrade\r\nUpgrade: websocket\r\n" - "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n") - s.sendall(request.encode()) - response = b"" - while True: - chunk = s.recv(4096) - if not chunk: - break - response += chunk - s.close() - assert response.startswith(b"HTTP/1.1 403"), response - assert b"unsupported_l7_protocol" in response, response - assert b"REST endpoints" in response, response -print("REST_UPGRADE_REFUSED") diff --git a/tests/suites/conformance/src/scripts/status_client.py b/tests/suites/conformance/src/scripts/status_client.py deleted file mode 100644 index 1d059a9c37..0000000000 --- a/tests/suites/conformance/src/scripts/status_client.py +++ /dev/null @@ -1,44 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Sends two GET requests to argv[1]:argv[2], each on its own connection, and -# prints their statuses, or the error a connection failed with. -import json -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) - - -def read_headers(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - - -def status(response): - parts = response.split(None, 2) - return int(parts[1]) if len(parts) > 1 else 0 - - -def request_status(path): - try: - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall( - f"GET {path} HTTP/1.1\r\n" - f"Host: {HOST}:{PORT}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - except OSError as error: - return {"errno": error.errno, "error": repr(error)} - - -print(json.dumps({ - "first": request_status("/first"), - "second": request_status("/second"), -}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/uninspectable_client.py b/tests/suites/conformance/src/scripts/uninspectable_client.py deleted file mode 100644 index 75fa7d65a7..0000000000 --- a/tests/suites/conformance/src/scripts/uninspectable_client.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Sends bytes that are neither HTTP nor TLS to argv[1]:argv[2] and fails if -# the connection answers with anything but a 403 unsupported_l7_protocol -# denial. A reset or an empty close also counts as blocked. -import socket -import sys - -HOST = sys.argv[1] -PORT = int(sys.argv[2]) -PAYLOAD = bytes([0x00, 0xFF, 0x13, 0x37]) + b"not-http-or-tls" - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - denial = b"" - while True: - try: - chunk = sock.recv(4096) - except ConnectionResetError: - break - if not chunk: - break - denial += chunk -if denial and ( - b"HTTP/1.1 403 Forbidden" not in denial - or b"unsupported_l7_protocol" not in denial -): - raise SystemExit(f"missing the middleware denial: {denial!r}") -print("BLOCKED") diff --git a/tests/suites/conformance/src/support.rs b/tests/suites/conformance/src/support.rs deleted file mode 100644 index 5b80d1e5a6..0000000000 --- a/tests/suites/conformance/src/support.rs +++ /dev/null @@ -1,87 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Helpers the network and policy suites share. Not a suite. -//! -//! Their clients are Python scripts in `src/scripts/` that take the server's -//! host and port as arguments, run in a long-running sandbox from -//! [`testkit::python_image`], and print their result as one JSON line. - -use serde_json::Value; -use testkit::sdk::raw::proto; -use testkit::{Context, ExecOutput, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; - -/// The name sandboxes reach the test host's servers by. -pub const TEST_HOST: &str = "host.openshell.internal"; - -/// A long-running sandbox from the Python image with `policy`, Ready. -pub async fn python_sandbox( - cx: &Context, - suffix: &str, - policy: proto::SandboxPolicy, -) -> Result { - cx.sandbox(suffix) - .policy(policy) - .image(testkit::python_image()) - .command(["sleep", "infinity"]) - .create_ready() - .await -} - -/// Run `script` with the image's Python and `args`, and fail unless it -/// exits 0. -pub async fn run_python(sandbox: &Sandbox, script: &str, args: &[&str]) -> Result { - let mut command = vec!["python3", "-c", script]; - command.extend_from_slice(args); - sandbox - .exec(command) - .await? - .success() - .context("run the client in the sandbox") -} - -/// The last line of `output`'s stdout that parses as JSON. -pub fn json_result(output: &ExecOutput) -> Result { - output - .stdout - .lines() - .rev() - .find_map(|line| serde_json::from_str(line.trim()).ok()) - .ok_or_else(|| { - testkit::Error::msg(format!( - "no JSON result in the client's output:\n{}\n{}", - output.stdout, output.stderr - )) - }) -} - -/// Fail unless each key in `expected` has its value in `result`. -pub fn expect_values>(result: &Value, expected: &[(K, Value)]) -> Result { - let wrong: Vec = expected - .iter() - .filter(|(key, value)| result[key.as_ref()] != *value) - .map(|(key, value)| { - let key = key.as_ref(); - format!("{key}: expected {value}, got {}", result[key]) - }) - .collect(); - testkit::ensure!(wrong.is_empty(), "{}\nresult: {result}", wrong.join("\n")); - Ok(()) -} - -/// The sandbox's policy revisions, from `policy list --output json`. -pub async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { - cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) - .await? - .success()? - .json() -} - -/// `binaries:` for policy YAML that lets the image's Python connect. -pub fn python_binaries_yaml() -> String { - let paths: Vec = PYTHON_BINARIES - .iter() - .map(|path| format!("{{path: \"{path}\"}}")) - .collect(); - format!("[{}]", paths.join(", ")) -} diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 99f2256d96..3a0b5a30b2 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -3121,7 +3121,7 @@ version = "0.0.0" dependencies = [ "host-server-support", "openshell-testkit", - "serde_json", + "sandbox-clients-support", ] [[package]] @@ -3263,6 +3263,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "httparse", + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/suites/features/routable-host-ip/Cargo.toml b/tests/suites/features/routable-host-ip/Cargo.toml index 0645be32e8..bda256ea16 100644 --- a/tests/suites/features/routable-host-ip/Cargo.toml +++ b/tests/suites/features/routable-host-ip/Cargo.toml @@ -13,5 +13,5 @@ publish = false [dependencies] host-server-support = { path = "../../../support/host-server" } +sandbox-clients-support = { path = "../../../support/sandbox-clients" } openshell-testkit = { path = "../../../testkit/testkit" } -serde_json = "1" diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs index be414b0f0a..15c2720768 100644 --- a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -12,17 +12,11 @@ //! Ported from the `proxy_egress_pipeline` e2e test of IP literals, which //! used two support containers on the e2e network instead. -use host_server_support::HttpRecorder; -use serde_json::Value; +use host_server_support::{HttpRecorder, TEST_HOST}; +use sandbox_clients_support::{Request, http}; use testkit::policy; use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; -/// Sends two GET requests to `argv[1]:argv[2]` and prints their statuses as -/// one JSON line. -const STATUS_CLIENT: &str = include_str!("../../../conformance/src/scripts/status_client.py"); - -const RESOLVE: &str = "import socket; print(socket.gethostbyname('host.openshell.internal'))"; - /// Requests reach a server by an IP literal endpoint with `allowed_ips` for /// that address, and another by an IP literal endpoint without it. #[testkit::test] @@ -37,10 +31,12 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul .create_ready() .await?; let ip = sandbox - .exec(["python3", "-c", RESOLVE]) - .await? - .success() - .context("resolve host.openshell.internal in the sandbox")? + .run_python( + "import socket, sys; print(socket.gethostbyname(sys.argv[1]))", + &[TEST_HOST], + ) + .await + .context(format!("resolve {TEST_HOST} in the sandbox"))? .stdout .trim() .to_string(); @@ -64,33 +60,14 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul policy::set(&sandbox, &ip_policy).await?; for (mode, server) in [("explicit allowed_ips", &explicit), ("implicit", &implicit)] { - let output = sandbox - .exec([ - "python3", - "-c", - STATUS_CLIENT, - &ip, - &server.port.to_string(), - ]) - .await? - .success() - .context(format!("run the {mode} client"))?; - let result: Value = output - .stdout - .lines() - .rev() - .find_map(|line| serde_json::from_str(line.trim()).ok()) - .ok_or_else(|| { - testkit::Error::msg(format!( - "no JSON result from the {mode} client:\n{}\n{}", - output.stdout, output.stderr - )) - })?; - testkit::ensure!( - result["first"] == 200 && result["second"] == 200, - "{mode} requests to {ip}:{} failed: {result}", - server.port - ); + for path in ["/first", "/second"] { + let reply = http(&sandbox, &ip, server.port, &Request::get(path)).await?; + testkit::ensure!( + reply.status() == Some(200), + "{mode} request to {ip}:{}{path} failed: {reply:?}", + server.port + ); + } } Ok(()) } diff --git a/tests/suites/conformance/src/scripts/bound_credential_client.sh b/tests/support/sandbox-clients/scripts/bound_credential_client.sh similarity index 100% rename from tests/suites/conformance/src/scripts/bound_credential_client.sh rename to tests/support/sandbox-clients/scripts/bound_credential_client.sh diff --git a/tests/suites/conformance/src/scripts/echo_server.py b/tests/support/sandbox-clients/scripts/echo_server.py similarity index 100% rename from tests/suites/conformance/src/scripts/echo_server.py rename to tests/support/sandbox-clients/scripts/echo_server.py diff --git a/tests/suites/conformance/src/scripts/graphql_client.py b/tests/support/sandbox-clients/scripts/graphql_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/graphql_client.py rename to tests/support/sandbox-clients/scripts/graphql_client.py diff --git a/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py b/tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/jsonrpc_audit_client.py rename to tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py diff --git a/tests/suites/conformance/src/scripts/jsonrpc_client.py b/tests/support/sandbox-clients/scripts/jsonrpc_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/jsonrpc_client.py rename to tests/support/sandbox-clients/scripts/jsonrpc_client.py diff --git a/tests/suites/conformance/src/scripts/listener_lifetime_probe.py b/tests/support/sandbox-clients/scripts/listener_lifetime_probe.py similarity index 100% rename from tests/suites/conformance/src/scripts/listener_lifetime_probe.py rename to tests/support/sandbox-clients/scripts/listener_lifetime_probe.py diff --git a/tests/suites/conformance/src/scripts/localhost_client.py b/tests/support/sandbox-clients/scripts/localhost_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/localhost_client.py rename to tests/support/sandbox-clients/scripts/localhost_client.py diff --git a/tests/suites/conformance/src/scripts/mcp_client.py b/tests/support/sandbox-clients/scripts/mcp_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/mcp_client.py rename to tests/support/sandbox-clients/scripts/mcp_client.py diff --git a/tests/suites/conformance/src/scripts/preconnect_owner.py b/tests/support/sandbox-clients/scripts/preconnect_owner.py similarity index 100% rename from tests/suites/conformance/src/scripts/preconnect_owner.py rename to tests/support/sandbox-clients/scripts/preconnect_owner.py diff --git a/tests/suites/conformance/src/scripts/reload_keep_alive_client.py b/tests/support/sandbox-clients/scripts/reload_keep_alive_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/reload_keep_alive_client.py rename to tests/support/sandbox-clients/scripts/reload_keep_alive_client.py diff --git a/tests/suites/conformance/src/scripts/socket_not_inherited.py b/tests/support/sandbox-clients/scripts/socket_not_inherited.py similarity index 100% rename from tests/suites/conformance/src/scripts/socket_not_inherited.py rename to tests/support/sandbox-clients/scripts/socket_not_inherited.py diff --git a/tests/suites/conformance/src/scripts/websocket_text_client.py b/tests/support/sandbox-clients/scripts/websocket_text_client.py similarity index 100% rename from tests/suites/conformance/src/scripts/websocket_text_client.py rename to tests/support/sandbox-clients/scripts/websocket_text_client.py From 083c9a6cce2640df59a3bf631fe85cad8e9a3919 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:32:06 -0800 Subject: [PATCH 27/63] test(conformance): use the shared sandbox clients in the network suites Replace conformance's support module and generic client scripts with testkit's run_python, last_json, policy readers, providers and binaries_yaml, and the connect, http and http_raw probes. Protocol and specialized clients move to the sandbox-clients scripts module. The pipelined request tests now count every response in the reply; the 403 follows the first response's body with no line break, so the old line-based count missed it. The policy update tests use the Python image, which has the sandbox user the base policy runs as. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 11 + tests/suites/conformance/Cargo.toml | 1 + .../conformance/src/credential_gating.rs | 248 ++++++++++-------- tests/suites/conformance/src/lib.rs | 1 - tests/suites/conformance/src/network.rs | 115 ++++---- .../suites/conformance/src/network_egress.rs | 221 +++++++++------- tests/suites/conformance/src/network_l7.rs | 161 +++++++++--- .../suites/conformance/src/network_l7/mcp.rs | 33 +-- tests/suites/conformance/src/policy_update.rs | 47 ++-- tests/suites/conformance/src/port_forward.rs | 3 +- .../scripts/websocket_binary_client.py | 40 +++ tests/support/sandbox-clients/src/lib.rs | 54 ++++ 12 files changed, 601 insertions(+), 334 deletions(-) create mode 100644 tests/support/sandbox-clients/scripts/websocket_binary_client.py diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 9187a4cc8e..0d17b24979 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -542,6 +542,7 @@ dependencies = [ "host-server-support", "openshell-crypto", "openshell-testkit", + "sandbox-clients-support", "serde_json", "tempfile", "tokio", @@ -3182,6 +3183,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-clients-support" +version = "0.0.0" +dependencies = [ + "httparse", + "openshell-testkit", + "serde", + "serde_json", +] + [[package]] name = "schannel" version = "0.1.29" diff --git a/tests/suites/conformance/Cargo.toml b/tests/suites/conformance/Cargo.toml index ddcd2c7502..1e36cae072 100644 --- a/tests/suites/conformance/Cargo.toml +++ b/tests/suites/conformance/Cargo.toml @@ -25,6 +25,7 @@ harness = false [dependencies] host-server-support = { path = "../../support/host-server" } +sandbox-clients-support = { path = "../../support/sandbox-clients" } openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } serde_json = "1" diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index f6ff4a0c9d..7ac6a43983 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -11,59 +11,21 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, Response, TEXT_REPLY, WebSocketRecorder}; +use host_server_support::{ + HttpRecorder, Request, Response, TEST_HOST, TEXT_REPLY, WebSocketRecorder, +}; +use sandbox_clients_support::{self as clients, http, scripts}; use serde_json::json; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; -use crate::support::{self, TEST_HOST, python_binaries_yaml}; - -const CREDENTIAL_REWRITE_CLIENT: &str = include_str!("scripts/credential_rewrite_client.py"); -const WEBSOCKET_TEXT_CLIENT: &str = include_str!("scripts/websocket_text_client.py"); -const BOUND_CREDENTIAL_CLIENT: &str = include_str!("scripts/bound_credential_client.sh"); const BASH: &str = "/usr/bin/bash"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -/// POSTs a body containing the provider's token to `TEST_HOST` on the port -/// in `argv[1]`, directly rather than through a proxy. -const BODY_CLIENT: &str = r#" -import os, sys, urllib.request -body = ("prefix-" + os.environ["E2E_GATING_TOKEN"] + "-suffix").encode() -request = urllib.request.Request(f"http://host.openshell.internal:{sys.argv[1]}/token", data=body, headers={"Content-Type": "text/plain"}) -urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request, timeout=10).read() -"#; - -/// Upgrades to WebSocket at `TEST_HOST` on the port in `argv[1]`, sends one -/// binary frame, and waits briefly for the connection to be closed. -const WEBSOCKET_CLIENT: &str = r#" -import base64, os, socket, sys -host, port = "host.openshell.internal", int(sys.argv[1]) -with socket.create_connection((host, port), timeout=10) as sock: - key = base64.b64encode(os.urandom(16)).decode() - sock.sendall(f"GET /ws HTTP/1.1\r\nHost: {host}:{port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - chunk = sock.recv(4096) - if not chunk: - sys.exit(f"closed during the upgrade: {response!r}") - response += chunk - if not response.startswith(b"HTTP/1.1 101"): - sys.exit(f"upgrade failed: {response!r}") - payload, mask = b"binary-credential-channel", os.urandom(4) - sock.sendall(bytes([0x82, 0x80 | len(payload)]) + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) - sock.settimeout(3) - try: - sock.recv(4096) - except (TimeoutError, ConnectionError): - pass -"#; - -/// Prints `TOKEN_ABSENT`, `TOKEN_PLACEHOLDER`, or `TOKEN_UNSAFE` for the -/// provider's environment variable, without printing its value. const TOKEN_PRESENCE: &str = r#"if [ "${E2E_GATING_TOKEN+x}" != x ]; then echo TOKEN_ABSENT else @@ -82,11 +44,17 @@ async fn profile_rest_endpoint_keeps_body_placeholder(cx: &mut Context) -> testk let provider = install_gating_provider(cx, Some((server.port, websocket.port))).await?; let mut policy = policy::base(); - allow(&mut policy, server.port) - .rest(Access::Full) - .request_body_credential_rewrite(false); + policy::allow( + &mut policy, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .rest(Access::Full) + .request_body_credential_rewrite(false); let sandbox = create_sandbox(cx, "rest", policy, &[&provider]).await?; - run_python(&sandbox, BODY_CLIENT, server.port).await?; + post_token_body(&sandbox, server.port).await?; expect_placeholder_body(&server).await } @@ -124,25 +92,44 @@ async fn policy_binding_requires_inspection_or_opt_in(cx: &mut Context) -> testk let provider = install_gating_provider(cx, None).await?; let mut l4 = policy::base(); - allow(&mut l4, server.port).credential_binding(&provider); + policy::allow( + &mut l4, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .credential_binding(&provider); // The wording checks follow the #5264 tests. let error = expect_rejected(cx, "l4", l4, &provider).await?; testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); testkit::ensure!(error.contains("L4-only"), "{error}"); let mut tls_skip = policy::base(); - allow(&mut tls_skip, server.port) - .rest(Access::Full) - .tls_skip() - .credential_binding(&provider); + policy::allow( + &mut tls_skip, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .rest(Access::Full) + .tls_skip() + .credential_binding(&provider); let error = expect_rejected(cx, "tls", tls_skip, &provider).await?; testkit::ensure!(error.contains("credentialed endpoint"), "{error}"); testkit::ensure!(error.contains("tls:") && error.contains("skip"), "{error}"); let mut opt_in = policy::base(); - allow(&mut opt_in, server.port) - .allow_uninspected_credentials() - .credential_binding(&provider); + policy::allow( + &mut opt_in, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .allow_uninspected_credentials() + .credential_binding(&provider); let sandbox = create_sandbox(cx, "optin", opt_in, &[&provider]).await?; sandbox.exec(["true"]).await?.success()?; Ok(()) @@ -158,12 +145,18 @@ async fn policy_binding_rest_body_keeps_placeholder_without_rewrite( let provider = install_gating_provider(cx, None).await?; let mut policy = policy::base(); - allow(&mut policy, server.port) - .rest(Access::Full) - .request_body_credential_rewrite(false) - .credential_binding(&provider); + policy::allow( + &mut policy, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .rest(Access::Full) + .request_body_credential_rewrite(false) + .credential_binding(&provider); let sandbox = create_sandbox(cx, "body", policy, &[&provider]).await?; - run_python(&sandbox, BODY_CLIENT, server.port).await?; + post_token_body(&sandbox, server.port).await?; expect_placeholder_body(&server).await } @@ -177,12 +170,18 @@ async fn policy_binding_rest_body_rewrite_sends_secret_upstream( let provider = install_gating_provider(cx, None).await?; let mut policy = policy::base(); - allow(&mut policy, server.port) - .rest(Access::Full) - .request_body_credential_rewrite(true) - .credential_binding(&provider); + policy::allow( + &mut policy, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .rest(Access::Full) + .request_body_credential_rewrite(true) + .credential_binding(&provider); let sandbox = create_sandbox(cx, "rw", policy, &[&provider]).await?; - run_python(&sandbox, BODY_CLIENT, server.port).await?; + post_token_body(&sandbox, server.port).await?; let request = one_request(&server).await?; testkit::ensure!( request.body == format!("prefix-{TEST_SECRET}-suffix"), @@ -201,9 +200,15 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t let policy_binding = |bind: bool| { let mut policy = policy::base(); - let endpoint = allow(&mut policy, server.port) - .rest(Access::Full) - .request_body_credential_rewrite(false); + let endpoint = policy::allow( + &mut policy, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .rest(Access::Full) + .request_body_credential_rewrite(false); if bind { endpoint.credential_binding(&provider); } @@ -242,13 +247,25 @@ async fn profile_rest_endpoint_rewrites_header_and_body(cx: &mut Context) -> tes ), ) .await?; - support::run_python( - &sandbox, - CREDENTIAL_REWRITE_CLIENT, - &[TEST_HOST, &server.port.to_string()], - ) - .await? - .expect_stdout("REWRITE_OK\n")?; + let token = provider_token(&sandbox).await?; + let body = json!({ "credential": token }).to_string(); + for _ in 0..2 { + let reply = http( + &sandbox, + TEST_HOST, + server.port, + &clients::Request::new("POST", "/probe") + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body(body.clone()), + ) + .await?; + testkit::ensure!( + reply.status() == Some(200), + "expected 200; got {:?}", + reply.status() + ); + } let requests = server.wait_for_requests(2).await?; let authorization = format!("Bearer {TEST_SECRET}"); @@ -281,13 +298,13 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te ), ) .await?; - support::run_python( - &sandbox, - WEBSOCKET_TEXT_CLIENT, - &[TEST_HOST, &server.port.to_string()], - ) - .await? - .expect_stdout(&format!("{TEXT_REPLY}\n"))?; + sandbox + .run_python( + scripts::WEBSOCKET_TEXT_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout(&format!("{TEXT_REPLY}\n"))?; let texts = server.texts(); testkit::ensure!( @@ -343,7 +360,7 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: .sandbox("bind") .image(testkit::python_image()) .policy(policy) - .spec(|spec| spec.providers = vec![provider_a, provider_b]) + .providers([provider_a, provider_b]) .command(["sleep", "infinity"]) .create_ready() .await?; @@ -351,7 +368,7 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: .exec([ BASH, "-c", - BOUND_CREDENTIAL_CLIENT, + scripts::BOUND_CREDENTIAL_CLIENT, "bash", TEST_HOST, &server_a.port.to_string(), @@ -381,7 +398,7 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: Ok(()) } -/// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, +/// Run [`scripts::WEBSOCKET_BINARY_CLIENT`] against `server` in a sandbox with `providers`, /// and return whether its binary frame reached `server`. async fn send_binary_frame( cx: &Context, @@ -390,9 +407,21 @@ async fn send_binary_frame( providers: &[&str], ) -> Result { let mut policy = policy::base(); - allow(&mut policy, server.port).websocket(Access::ReadWrite); + policy::allow( + &mut policy, + "credential_gating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .websocket(Access::ReadWrite); let sandbox = create_sandbox(cx, suffix, policy, providers).await?; - run_python(&sandbox, WEBSOCKET_CLIENT, server.port).await?; + sandbox + .run_python( + scripts::WEBSOCKET_BINARY_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await?; server .wait_for_handshake() .await @@ -400,17 +429,6 @@ async fn send_binary_frame( server.wait_until_finished().await } -/// Let the image's Python reach `TEST_HOST` on `port`. -fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { - policy::allow( - policy, - "credential_gating", - TEST_HOST, - port, - PYTHON_BINARIES, - ) -} - /// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`. With /// `endpoints`, its profile declares the REST and WebSocket ports as its /// endpoints; without, a policy must bind the provider explicitly. @@ -438,7 +456,7 @@ async fn profile_endpoint_sandbox(cx: &Context, suffix: &str, endpoint: &str) -> endpoints: [{endpoint}] binaries: {binaries} ", - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; create_sandbox(cx, suffix, policy, &[&provider]).await } @@ -514,7 +532,7 @@ fn sandbox_with( cx.sandbox(suffix) .policy(policy) .image(testkit::python_image()) - .spec(|spec| spec.providers = providers.iter().map(ToString::to_string).collect()) + .providers(providers.iter().copied()) .command(["sleep", "infinity"]) } @@ -551,14 +569,34 @@ async fn expect_rejected( } } -/// Run `script` with the image's Python and `port` as its argument. -async fn run_python(sandbox: &Sandbox, script: &str, port: u16) -> Result { - let port = port.to_string(); - sandbox - .exec(["python3", "-c", script, &port]) +/// The provider token's placeholder, as the sandbox's environment holds it. +async fn provider_token(sandbox: &Sandbox) -> Result { + let output = sandbox + .exec(["printenv", TOKEN_ENV]) .await? .success() - .context("run the client in the sandbox")?; + .context("read the provider token from the sandbox environment")?; + Ok(output.stdout.trim().to_string()) +} + +/// POST `prefix--suffix` to `/token` on `TEST_HOST` at `port`, and +/// fail unless it gets `200`. +async fn post_token_body(sandbox: &Sandbox, port: u16) -> Result { + let token = provider_token(sandbox).await?; + let reply = http( + sandbox, + TEST_HOST, + port, + &clients::Request::new("POST", "/token") + .header("Content-Type", "text/plain") + .body(format!("prefix-{token}-suffix")), + ) + .await?; + testkit::ensure!( + reply.status() == Some(200), + "expected 200; got {:?}", + reply.status() + ); Ok(()) } diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 8bd5846c89..e3a4726e9e 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -29,6 +29,5 @@ mod process; mod settings; mod smoke; mod ssh; -mod support; mod templates; mod workspaces; diff --git a/tests/suites/conformance/src/network.rs b/tests/suites/conformance/src/network.rs index fdbc8c544f..8087135bdf 100644 --- a/tests/suites/conformance/src/network.rs +++ b/tests/suites/conformance/src/network.rs @@ -10,35 +10,33 @@ use std::time::Duration; +use sandbox_clients_support::{connect, scripts}; use serde_json::json; -use testkit::{Context, ResultExt as _, policy}; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _, policy}; -use crate::support::{ - expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, -}; - -const BYPASS_CLIENT: &str = include_str!("scripts/bypass_client.py"); -const DESTINATION_DENIALS_CLIENT: &str = include_str!("scripts/destination_denials_client.py"); -const LOCALHOST_CLIENT: &str = include_str!("scripts/localhost_client.py"); -const LISTENER_LIFETIME_PROBE: &str = include_str!("scripts/listener_lifetime_probe.py"); -const PRECONNECT_OWNER: &str = include_str!("scripts/preconnect_owner.py"); -const SOCKET_NOT_INHERITED: &str = include_str!("scripts/socket_not_inherited.py"); +/// `EACCES`, the errno the sandbox denies a reserved destination with. +const EACCES: i32 = 13; /// A direct TCP connection that bypasses the proxy is denied with a /// permission error well before the socket's 10-second timeout. #[testkit::test] async fn direct_connect_bypass_is_denied_promptly(cx: &mut Context) -> testkit::Result { - let sandbox = python_sandbox(cx, "bypas", policy::base()).await?; - let output = run_python(&sandbox, BYPASS_CLIENT, &[]).await?; - let result = json_result(&output)?; + let sandbox = cx + .sandbox("bypas") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy::base()) + .create_ready() + .await?; + let probe = connect(&sandbox, "198.51.100.1", 80).await?; testkit::ensure!( - result["bypass_result"] == "denied", - "expected the direct connect to be denied; got {result}" + probe.outcome.is_permission_denied(), + "expected the direct connect to be denied; got {probe:?}" ); - let elapsed = result["elapsed_ms"].as_u64().unwrap_or(u64::MAX); testkit::ensure!( - elapsed < 8000, - "the denial took {elapsed} ms; expected under 8000" + probe.elapsed < Duration::from_secs(8), + "the denial took {:?}; expected under 8 s", + probe.elapsed ); Ok(()) } @@ -63,33 +61,49 @@ async fn reserved_destinations_fail_with_eacces_even_when_allowed( allowed_ips: ["198.51.100.0/24"] binaries: {binaries} "#, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; - let sandbox = python_sandbox(cx, "deny", policy).await?; - let output = run_python(&sandbox, DESTINATION_DENIALS_CLIENT, &[]).await?; - expect_values( - &json_result(&output)?, - &[ - ("metadata", json!(13)), - ("control_plane", json!(13)), - ("outside_allowed_ips", json!(13)), - ], - ) + let sandbox = cx + .sandbox("deny") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + for (destination, host, port) in [ + ("metadata", "169.254.169.254", 80), + ("control_plane", "203.0.113.10", 6443), + ("outside_allowed_ips", "203.0.113.10", 8080), + ] { + let probe = connect(&sandbox, host, port).await?; + testkit::ensure!( + probe.outcome.errno() == Some(EACCES), + "expected EACCES for {destination}; got {probe:?}" + ); + } + Ok(()) } /// A process can serve and fetch HTTP on 127.0.0.1 without proxy variables /// in its environment. #[testkit::test] async fn localhost_is_reachable_without_proxy_environment(cx: &mut Context) -> testkit::Result { - let sandbox = python_sandbox(cx, "local", policy::base()).await?; - let output = run_python(&sandbox, LOCALHOST_CLIENT, &[]).await?; - expect_values( - &json_result(&output)?, - &[ - ("proxy_env_absent", json!(true)), - ("payload", json!({"message": "hello"})), - ], - ) + let sandbox = cx + .sandbox("local") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy::base()) + .create_ready() + .await?; + let result = sandbox + .run_python(scripts::LOCALHOST_CLIENT, &[]) + .await? + .last_json()?; + testkit::ensure!( + result["proxy_env_absent"] == true && result["payload"] == json!({"message": "hello"}), + "expected a local fetch without proxy variables; got {result}" + ); + Ok(()) } /// Closing a listener, or exiting while holding one, frees its port: new @@ -97,10 +111,20 @@ async fn localhost_is_reachable_without_proxy_environment(cx: &mut Context) -> t /// doesn't inherit the exited child's listener. #[testkit::test] async fn closed_or_exited_listener_releases_its_port(cx: &mut Context) -> testkit::Result { - let sandbox = python_sandbox(cx, "lsten", policy::base()).await?; - let output = run_python(&sandbox, LISTENER_LIFETIME_PROBE, &[]).await?; - let inode = json_result(&output)?[1].to_string(); - run_python(&sandbox, SOCKET_NOT_INHERITED, &[&inode]) + let sandbox = cx + .sandbox("lsten") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy::base()) + .create_ready() + .await?; + let inode = sandbox + .run_python(scripts::LISTENER_LIFETIME_PROBE, &[]) + .await? + .last_json()?[1] + .to_string(); + sandbox + .run_python(scripts::SOCKET_NOT_INHERITED, &[&inode]) .await .context("a later exec inherited the closed listener")?; Ok(()) @@ -114,7 +138,7 @@ async fn exec_does_not_inherit_main_process_socket(cx: &mut Context) -> testkit: .sandbox("prcon") .policy(policy::base()) .image(testkit::python_image()) - .command(["python3", "-c", PRECONNECT_OWNER]) + .command(["python3", "-c", scripts::PRECONNECT_OWNER]) .create_ready() .await?; let inode = sandbox @@ -126,7 +150,8 @@ async fn exec_does_not_inherit_main_process_socket(cx: &mut Context) -> testkit: ) .await? .stdout; - run_python(&sandbox, SOCKET_NOT_INHERITED, &[inode.trim()]) + sandbox + .run_python(scripts::SOCKET_NOT_INHERITED, &[inode.trim()]) .await .context("the exec inherited the main process's socket")?; Ok(()) diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 899e146eee..85ff744caf 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -12,25 +12,14 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, Response, TcpServer}; -use serde_json::{Value, json}; +use host_server_support::{HttpRecorder, Request, Response, TEST_HOST, TcpServer}; +use sandbox_clients_support::{self as clients, http, http_raw, scripts}; +use serde_json::Value; use testkit::policy::{self, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; -use crate::support::{ - TEST_HOST, expect_values, json_result, policy_revisions, python_binaries_yaml, python_sandbox, - run_python, -}; - -const STATUS_CLIENT: &str = include_str!("scripts/status_client.py"); -const RELOAD_KEEP_ALIVE_CLIENT: &str = include_str!("scripts/reload_keep_alive_client.py"); -const ECHO_CLIENT: &str = include_str!("scripts/echo_client.py"); -const UNINSPECTABLE_CLIENT: &str = include_str!("scripts/uninspectable_client.py"); -const REDACTION_CLIENT: &str = include_str!("scripts/redaction_client.py"); -const PIPELINE_CLIENT: &str = include_str!("scripts/pipeline_client.py"); - -/// Files [`RELOAD_KEEP_ALIVE_CLIENT`] signals through. +/// Files [`scripts::RELOAD_KEEP_ALIVE_CLIENT`] signals through. const RELOAD_READY: &str = "/tmp/proxy-reload-ready"; const RELOAD_GO: &str = "/tmp/proxy-reload-go"; const RELOAD_RESULT: &str = "/tmp/proxy-reload-result"; @@ -47,7 +36,13 @@ const PIPELINE_REPLY: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"; async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; let allowed = allow_server(server.port); - let sandbox = python_sandbox(cx, "reld", allowed.clone()).await?; + let sandbox = cx + .sandbox("reld") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(allowed.clone()) + .create_ready() + .await?; policy::set(&sandbox, &allowed).await?; let port = server.port.to_string(); @@ -57,13 +52,13 @@ async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> test "-c", &format!("nohup python3 -c \"$1\" {TEST_HOST} {port} >{RELOAD_LOG} 2>&1 &"), "reload-client", - RELOAD_KEEP_ALIVE_CLIENT, + scripts::RELOAD_KEEP_ALIVE_CLIENT, ]) .await? .success() .context("start the keep-alive client")?; wait_for_file(&sandbox, RELOAD_READY).await?; - expect_statuses(&sandbox, &port, |status| status == 200) + expect_statuses(&sandbox, server.port, |status| status == Some(200)) .await .context("before the reload")?; @@ -79,7 +74,7 @@ async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> test result["failed_closed"] == true, "the open connection forwarded a request after the reload: {result}" ); - expect_statuses(&sandbox, &port, |status| status != 200) + expect_statuses(&sandbox, server.port, |status| status != Some(200)) .await .context("after the reload") } @@ -93,13 +88,18 @@ async fn ambiguous_policy_update_is_rejected_and_active_policy_kept( ) -> testkit::Result { let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; let allowed = allow_server(server.port); - let sandbox = python_sandbox(cx, "ambig", allowed.clone()).await?; + let sandbox = cx + .sandbox("ambig") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(allowed.clone()) + .create_ready() + .await?; policy::set(&sandbox, &allowed).await?; - let port = server.port.to_string(); - expect_statuses(&sandbox, &port, |status| status == 200) + expect_statuses(&sandbox, server.port, |status| status == Some(200)) .await .context("before the rejected update")?; - let revisions = policy_revisions(cx, &sandbox).await?; + let revisions = sandbox.policy_revisions().await?; let connections = server.connections(); let mut ambiguous = policy::base(); @@ -138,12 +138,12 @@ async fn ambiguous_policy_update_is_rejected_and_active_policy_kept( output.report_text() ); - let after = policy_revisions(cx, &sandbox).await?; + let after = sandbox.policy_revisions().await?; testkit::ensure!( after == revisions, "the rejected policy changed the revisions:\nbefore: {revisions}\nafter: {after}" ); - expect_statuses(&sandbox, &port, |status| status == 200) + expect_statuses(&sandbox, server.port, |status| status == Some(200)) .await .context("after the rejected update")?; testkit::ensure!( @@ -160,18 +160,17 @@ async fn tls_skip_relays_opaque_bytes(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let mut policy = policy::base(); allow(&mut policy, server.port).tls_skip(); - let sandbox = python_sandbox(cx, "tlskp", policy).await?; + let sandbox = cx + .sandbox("tlskp") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]; payload.extend_from_slice(b"not-http-or-tls"); payload.extend(0..64_u8); - run_python( - &sandbox, - ECHO_CLIENT, - &[TEST_HOST, &server.port.to_string(), &hex(&payload)], - ) - .await? - .expect_stdout("ECHO_OK\n")?; - Ok(()) + expect_echo(&sandbox, server.port, &payload).await } /// A redacting middleware rewrites an API key in each request body before @@ -185,17 +184,32 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result }) .await?; let policy = middleware_policy(server.port, "fail_closed", "")?; - let sandbox = python_sandbox(cx, "redct", policy).await?; - let output = run_python( - &sandbox, - REDACTION_CLIENT, - &[TEST_HOST, &server.port.to_string()], - ) - .await?; - let result = json_result(&output)?; + let sandbox = cx + .sandbox("redct") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; for request in ["first", "second"] { - expect_values(&result[request], &[("api_key", json!("[REDACTED]"))]) - .context(format!("the {request} request"))?; + let reply = http( + &sandbox, + TEST_HOST, + server.port, + &clients::Request::new("POST", "/middleware") + .header("Content-Type", "application/json") + .body(r#"{"api_key":"sk-1234567890abcdef"}"#), + ) + .await?; + let echoed: Option = reply + .response + .as_ref() + .and_then(|response| serde_json::from_slice(&response.body).ok()); + testkit::ensure!( + reply.status() == Some(200) + && echoed.is_some_and(|body| body["api_key"] == "[REDACTED]"), + "the {request} request reached upstream unredacted: {reply:?}" + ); } Ok(()) } @@ -206,14 +220,24 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_closed", "")?; - let sandbox = python_sandbox(cx, "fclos", policy).await?; - run_python( - &sandbox, - UNINSPECTABLE_CLIENT, - &[TEST_HOST, &server.port.to_string()], - ) - .await? - .expect_stdout("BLOCKED\n")?; + let sandbox = cx + .sandbox("fclos") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + let mut payload = vec![0x00, 0xff, 0x13, 0x37]; + payload.extend_from_slice(b"not-http-or-tls"); + let reply = http_raw(&sandbox, TEST_HOST, server.port, &payload).await?; + let denial = reply.received_text(); + testkit::ensure!( + reply.connect.is_connected() + && (denial.is_empty() + || denial.contains("HTTP/1.1 403 Forbidden") + && denial.contains("unsupported_l7_protocol")), + "expected the middleware to block the payload: {reply:?}" + ); // Give a forwarded payload time to arrive before checking it didn't. tokio::time::sleep(Duration::from_millis(200)).await; let received = server.received(); @@ -239,16 +263,16 @@ async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) - async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> testkit::Result { let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_open", " tls: skip\n")?; - let sandbox = python_sandbox(cx, "fopen", policy).await?; + let sandbox = cx + .sandbox("fopen") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80]; payload.extend_from_slice(b"middleware-bypass"); - run_python( - &sandbox, - ECHO_CLIENT, - &[TEST_HOST, &server.port.to_string(), &hex(&payload)], - ) - .await? - .expect_stdout("ECHO_OK\n")?; + expect_echo(&sandbox, server.port, &payload).await?; let received = server.received(); testkit::ensure!( received == payload, @@ -264,7 +288,13 @@ async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> #[testkit::test] async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; - let sandbox = python_sandbox(cx, "pipe", rest_policy(server.port, "GET")?).await?; + let sandbox = cx + .sandbox("pipe") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(rest_policy(server.port, "GET")?) + .create_ready() + .await?; let target = format!("{TEST_HOST}:{}", server.port); let requests = format!( "GET /allowed HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n\ @@ -287,7 +317,13 @@ async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit #[testkit::test] async fn chunked_pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; - let sandbox = python_sandbox(cx, "chunk", rest_policy(server.port, "POST")?).await?; + let sandbox = cx + .sandbox("chunk") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(rest_policy(server.port, "POST")?) + .create_ready() + .await?; let target = format!("{TEST_HOST}:{}", server.port); let requests = format!( "POST /allowed HTTP/1.1\r\nHost: {target}\r\nTransfer-Encoding: chunked\r\n\ @@ -344,7 +380,7 @@ network_policies: port: {port} {endpoint_options} binaries: {binaries} "#, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), )) } @@ -365,50 +401,51 @@ fn rest_policy(port: u16, method: &str) -> Result { path: /allowed binaries: {binaries} ", - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), )) } /// Send `requests` pipelined in one write, and fail unless the first gets /// `200` and the second `403`. async fn run_pipeline(sandbox: &Sandbox, port: u16, requests: &str) -> Result { - let output = run_python( - sandbox, - PIPELINE_CLIENT, - &[TEST_HOST, &port.to_string(), requests], - ) - .await?; - let result = json_result(&output)?; - let statuses: Vec<&str> = result["statuses"] - .as_array() - .into_iter() - .flatten() - .filter_map(Value::as_str) - .collect(); + let reply = http_raw(sandbox, TEST_HOST, port, requests.as_bytes()).await?; + let received = reply.received_text(); testkit::ensure!( - statuses.len() == 2 - && statuses[0].starts_with("HTTP/1.1 200") - && statuses.contains(&"HTTP/1.1 403 Forbidden"), - "expected 200 then 403; got {statuses:?}" + reply.status() == Some(200) + && received.matches("HTTP/1.1 ").count() == 2 + && received.contains("HTTP/1.1 403 Forbidden"), + "expected 200 then 403; got {reply:?}" ); Ok(()) } -/// Run [`STATUS_CLIENT`] and fail unless both its requests got a status -/// `expected` accepts. A failed connection counts as status 0. -async fn expect_statuses(sandbox: &Sandbox, port: &str, expected: impl Fn(i64) -> bool) -> Result { - let output = run_python(sandbox, STATUS_CLIENT, &[TEST_HOST, port]).await?; - let result = json_result(&output)?; - for request in ["first", "second"] { - let status = result[request].as_i64().unwrap_or(0); +/// Send `GET /first` and `GET /second` on new connections, and fail unless +/// `expected` accepts both statuses. A request without a response is `None`. +async fn expect_statuses( + sandbox: &Sandbox, + port: u16, + expected: impl Fn(Option) -> bool, +) -> Result { + for path in ["/first", "/second"] { + let reply = http(sandbox, TEST_HOST, port, &clients::Request::get(path)).await?; testkit::ensure!( - expected(status), - "unexpected {request} request outcome: {result}" + expected(reply.status()), + "unexpected outcome for {path}: {reply:?}" ); } Ok(()) } +/// Send `payload` and fail unless the same bytes come back. +async fn expect_echo(sandbox: &Sandbox, port: u16, payload: &[u8]) -> Result { + let reply = http_raw(sandbox, TEST_HOST, port, payload).await?; + testkit::ensure!( + reply.received == payload, + "the payload changed in transit: {reply:?}" + ); + Ok(()) +} + /// The contents of `path` in the sandbox, once it exists. A timeout reports /// the keep-alive client's log. async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { @@ -426,7 +463,3 @@ async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { .await? .stdout) } - -fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs index bbf71fde51..30c6845d3c 100644 --- a/tests/suites/conformance/src/network_l7.rs +++ b/tests/suites/conformance/src/network_l7.rs @@ -12,31 +12,33 @@ mod mcp; -use host_server_support::{HttpRecorder, Request, Response}; -use serde_json::json; -use testkit::{Context, Result, ResultExt as _, policy}; - -use crate::support::{ - TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, -}; - -const REST_CLIENT: &str = include_str!("scripts/rest_client.py"); -const REST_UPGRADE_CLIENT: &str = include_str!("scripts/rest_upgrade_client.py"); -const GRAPHQL_CLIENT: &str = include_str!("scripts/graphql_client.py"); -const JSONRPC_CLIENT: &str = include_str!("scripts/jsonrpc_client.py"); -const JSONRPC_AUDIT_CLIENT: &str = include_str!("scripts/jsonrpc_audit_client.py"); +use host_server_support::{HttpRecorder, Request, Response, TEST_HOST}; +use sandbox_clients_support::{self as clients, http, http_raw, scripts}; +use serde_json::{Value, json}; +use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, policy}; /// A REST endpoint that allows only `GET /allowed` passes that request and /// answers a `POST` to the same path with `403` before it reaches upstream. #[testkit::test] async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(rest_upstream).await?; - let sandbox = python_sandbox(cx, "rest", rest_policy(server.port, "enforce")?).await?; - let output = run_python(&sandbox, REST_CLIENT, &[TEST_HOST, &port(&server)]).await?; - expect_values( - &json_result(&output)?, - &[("get", json!(200)), ("post", json!(403))], - )?; + let sandbox = cx + .sandbox("rest") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(rest_policy(server.port, "enforce")?) + .create_ready() + .await?; + for (request, expected) in [ + (clients::Request::get("/allowed"), 200), + (clients::Request::new("POST", "/allowed").body("test"), 403), + ] { + let reply = http(&sandbox, TEST_HOST, server.port, &request).await?; + testkit::ensure!( + reply.status() == Some(expected), + "expected {expected} for {request:?}; got {reply:?}" + ); + } let methods: Vec = server .requests() .iter() @@ -56,11 +58,16 @@ async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result async fn rest_upgrade_is_refused_in_enforce_and_audit_modes(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(rest_upstream).await?; for (suffix, enforcement) in [("enfrc", "enforce"), ("audit", "audit")] { - let sandbox = python_sandbox(cx, suffix, rest_policy(server.port, enforcement)?).await?; - run_python(&sandbox, REST_UPGRADE_CLIENT, &[TEST_HOST, &port(&server)]) + let sandbox = cx + .sandbox(suffix) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(rest_policy(server.port, enforcement)?) + .create_ready() + .await?; + expect_upgrade_refused(&sandbox, server.port) .await - .context(format!("{enforcement} mode"))? - .expect_stdout("REST_UPGRADE_REFUSED\n")?; + .context(format!("{enforcement} mode"))?; } Ok(()) } @@ -106,10 +113,18 @@ async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::R binaries: {binaries} ", port = server.port, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; - let sandbox = python_sandbox(cx, "gql", policy).await?; - let output = run_python(&sandbox, GRAPHQL_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let sandbox = cx + .sandbox("gql") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + let output = sandbox + .run_python(scripts::GRAPHQL_CLIENT, &[TEST_HOST, &port(&server)]) + .await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { for (key, status) in [ @@ -123,10 +138,10 @@ async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::R ("mutation_allowed", 200), ("deny_rule_denied", 403), ] { - expected.push((format!("{path}_{key}"), json!(status))); + expected.push((format!("{path}_{key}"), status)); } } - expect_values(&json_result(&output)?, &expected) + expect_statuses(&output, expected) } /// JSON-RPC rules allow listed methods whatever their params, deny a @@ -159,10 +174,18 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu binaries: {binaries} ", port = server.port, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; - let sandbox = python_sandbox(cx, "jrpc", policy).await?; - let output = run_python(&sandbox, JSONRPC_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let sandbox = cx + .sandbox("jrpc") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + let output = sandbox + .run_python(scripts::JSONRPC_CLIENT, &[TEST_HOST, &port(&server)]) + .await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { for (key, status) in [ @@ -172,7 +195,7 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu ("method_tools_call_with_unmatched_params_allowed", 200), ("method_tools_delete_denied", 403), ] { - expected.push((format!("{path}_{key}"), json!(status))); + expected.push((format!("{path}_{key}"), status)); } } for (key, status) in [ @@ -180,9 +203,9 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu ("forward_batch_one_denied", 403), ("forward_invalid_json_denied", 403), ] { - expected.push((key.to_string(), json!(status))); + expected.push((key.to_string(), status)); } - expect_values(&json_result(&output)?, &expected) + expect_statuses(&output, expected) } /// A JSON-RPC endpoint without `enforcement` audits an unlisted method and @@ -209,15 +232,23 @@ async fn jsonrpc_default_audit_mode_still_denies_response_frames( binaries: {binaries} ", port = server.port, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; - let sandbox = python_sandbox(cx, "jaud", policy).await?; - let output = run_python(&sandbox, JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]).await?; - expect_values( - &json_result(&output)?, - &[ - ("forward_unknown_method_audited", json!(200)), - ("forward_response_frame_hard_denied", json!(403)), + let sandbox = cx + .sandbox("jaud") + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + let output = sandbox + .run_python(scripts::JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]) + .await?; + expect_statuses( + &output, + [ + ("forward_unknown_method_audited".to_string(), 200), + ("forward_response_frame_hard_denied".to_string(), 403), ], ) } @@ -240,7 +271,7 @@ fn rest_policy(port: u16, enforcement: &str) -> Result Response { } } +/// Send a WebSocket upgrade for `/allowed`, in origin and absolute form, +/// and fail unless the sandbox refuses each as unsupported on a REST +/// endpoint. +async fn expect_upgrade_refused(sandbox: &Sandbox, port: u16) -> Result { + for target in [ + "/allowed".to_string(), + format!("http://{TEST_HOST}:{port}/allowed"), + ] { + let request = format!( + "GET {target} HTTP/1.1\r\nHost: {TEST_HOST}:{port}\r\n\ + Connection: Upgrade\r\nUpgrade: websocket\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" + ); + let reply = http_raw(sandbox, TEST_HOST, port, request.as_bytes()).await?; + let received = reply.received_text(); + testkit::ensure!( + reply.status() == Some(403) + && received.contains("unsupported_l7_protocol") + && received.contains("REST endpoints"), + "expected the upgrade to {target} refused; got {reply:?}" + ); + } + Ok(()) +} + +/// Fail unless the client's JSON result maps exactly the keys in +/// `expected` to their statuses. +fn expect_statuses( + output: &testkit::ExecOutput, + expected: impl IntoIterator, +) -> Result { + let result = output.last_json()?; + let expected: Value = expected + .into_iter() + .map(|(key, status)| (key, json!(status))) + .collect::>() + .into(); + testkit::ensure!( + result == expected, + "unexpected statuses:\nexpected: {expected}\ngot: {result}" + ); + Ok(()) +} + fn ok_json() -> Response { Response::json(&json!({"ok": true})) } diff --git a/tests/suites/conformance/src/network_l7/mcp.rs b/tests/suites/conformance/src/network_l7/mcp.rs index c553516bb9..d9c19edd77 100644 --- a/tests/suites/conformance/src/network_l7/mcp.rs +++ b/tests/suites/conformance/src/network_l7/mcp.rs @@ -12,13 +12,10 @@ use std::sync::{Arc, Mutex, PoisonError}; -use host_server_support::{HttpRecorder, Request, Response}; +use host_server_support::{HttpRecorder, Request, Response, TEST_HOST}; +use sandbox_clients_support::scripts::MCP_CLIENT; use serde_json::{Value, json}; -use testkit::{Context, Result, ResultExt as _, policy}; - -use crate::support::{TEST_HOST, python_binaries_yaml, python_sandbox, run_python}; - -const MCP_CLIENT: &str = include_str!("../scripts/mcp_client.py"); +use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, policy}; const SESSIONLESS: &str = "2026-07-28"; /// The sessionless revision: discovery, an allowed and a denied tool, and a @@ -85,16 +82,22 @@ async fn run_profile(cx: &Context, suffix: &str, versions: &[&str]) -> Result { binaries: {binaries} ", port = server.port, - binaries = python_binaries_yaml(), + binaries = policy::binaries_yaml(PYTHON_BINARIES), ))?; - let sandbox = python_sandbox(cx, suffix, policy).await?; - let output = run_python( - &sandbox, - MCP_CLIENT, - &[TEST_HOST, &server.port.to_string(), &versions_json], - ) - .await - .context(format!("MCP revisions {versions:?}"))?; + let sandbox = cx + .sandbox(suffix) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .policy(policy) + .create_ready() + .await?; + let output = sandbox + .run_python( + MCP_CLIENT, + &[TEST_HOST, &server.port.to_string(), &versions_json], + ) + .await + .context(format!("MCP revisions {versions:?}"))?; for version in versions { let marker = format!("MCP_OK version={version}"); testkit::ensure!( diff --git a/tests/suites/conformance/src/policy_update.rs b/tests/suites/conformance/src/policy_update.rs index 4fe50706fd..73f611f0a1 100644 --- a/tests/suites/conformance/src/policy_update.rs +++ b/tests/suites/conformance/src/policy_update.rs @@ -15,8 +15,6 @@ use testkit::policy; use testkit::sdk::raw::proto; use testkit::{Context, Result, Sandbox}; -use crate::support::policy_revisions; - /// A network-only policy the supervisor fills in with the filesystem paths it /// needs at startup, which makes a second revision. const SPARSE_POLICY: &str = include_str!(concat!( @@ -35,11 +33,12 @@ const LOAD_TIMEOUT: Duration = Duration::from_mins(2); async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("pset") + .image(testkit::python_image()) .policy(policy::base()) .command(["sleep", "infinity"]) .create_ready() .await?; - let initial = settled(cx, &sandbox).await?; + let initial = settled(&sandbox).await?; let mut one_rule = policy::base(); policy::allow(&mut one_rule, "rule_0", "api.anthropic.com", 443, &["/**"]); @@ -50,7 +49,7 @@ async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::R let mut versions = vec![version(&initial)?]; for (step, next) in [("one rule", &one_rule), ("two rules", &two_rules)] { policy::set(&sandbox, next).await?; - let updated = settled(cx, &sandbox).await?; + let updated = settled(&sandbox).await?; testkit::ensure!( version(&updated)? > version(&previous)?, "setting {step} didn't make a newer version: before {previous}, after {updated}" @@ -62,7 +61,7 @@ async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::R ); policy::set(&sandbox, next).await?; - let repeated = settled(cx, &sandbox).await?; + let repeated = settled(&sandbox).await?; for field in ["version", "hash"] { testkit::ensure!( repeated[field] == updated[field], @@ -75,7 +74,7 @@ async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::R previous = updated; } - let revisions = policy_revisions(cx, &sandbox).await?; + let revisions = sandbox.policy_revisions().await?; let listed: Vec = revisions["revisions"] .as_array() .into_iter() @@ -98,11 +97,12 @@ async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::R async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> testkit::Result { let sandbox = cx .sandbox("pscop") + .image(testkit::python_image()) .policy(scoped_policy()?) .command(["sleep", "infinity"]) .create_ready() .await?; - let before = settled(cx, &sandbox).await?; + let before = settled(&sandbox).await?; let rules_before = &before["policy"]["network_policies"]; testkit::ensure!( rules_before.is_object(), @@ -134,7 +134,7 @@ async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> t args.extend(["--binary", binary]); } cx.cli(args).await?.failure()?; - let unchanged = policy_get(cx, &sandbox).await?; + let unchanged = policy_get(&sandbox).await?; for field in ["version", "hash", "policy"] { testkit::ensure!( unchanged[field] == before[field], @@ -150,7 +150,7 @@ async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> t "api.example.com:443,8443:POST:/admin", ) .await?; - let allowed = policy_get(cx, &sandbox).await?; + let allowed = policy_get(&sandbox).await?; testkit::ensure!( version(&allowed)? > version(&before)? && allowed["hash"] != before["hash"], "the allow didn't make a new revision: before {before}, after {allowed}" @@ -181,7 +181,7 @@ async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> t "api.example.com:443,8443:POST:/admin/private", ) .await?; - let denied = policy_get(cx, &sandbox).await?; + let denied = policy_get(&sandbox).await?; testkit::ensure!( version(&denied)? > version(&allowed)?, "the deny didn't make a new revision: before {allowed}, after {denied}" @@ -219,8 +219,8 @@ async fn sparse_initial_policy_revision_is_loaded(cx: &mut Context) -> testkit:: LOAD_TIMEOUT, Duration::from_secs(2), async || { - let active = policy_get(cx, &sandbox).await?; - let revisions = policy_revisions(cx, &sandbox).await?; + let active = policy_get(&sandbox).await?; + let revisions = sandbox.policy_revisions().await?; let statuses = statuses(&revisions); let expected = [(1_u64, "superseded".to_string()), (2, "loaded".to_string())]; Ok( @@ -280,20 +280,9 @@ async fn update(cx: &Context, sandbox: &Sandbox, flag: &str, rule: &str) -> Resu Ok(()) } -/// The sandbox's active policy, from `policy get --full --output json`. -async fn policy_get(cx: &Context, sandbox: &Sandbox) -> Result { - let active: Value = cx - .cli([ - "policy", - "get", - sandbox.name(), - "--full", - "--output", - "json", - ]) - .await? - .success()? - .json()?; +/// The sandbox's active policy, checked for a version and hash. +async fn policy_get(sandbox: &Sandbox) -> Result { + let active = sandbox.active_policy().await?; version(&active)?; testkit::ensure!( active["hash"].as_str().is_some_and(|hash| !hash.is_empty()), @@ -304,14 +293,14 @@ async fn policy_get(cx: &Context, sandbox: &Sandbox) -> Result { /// The active policy once its revision is loaded and no revision is pending. /// Fails if a revision failed to load. -async fn settled(cx: &Context, sandbox: &Sandbox) -> Result { +async fn settled(sandbox: &Sandbox) -> Result { testkit::poll_until( &format!("the policy of sandbox {} to be loaded", sandbox.name()), LOAD_TIMEOUT, Duration::from_secs(1), async || { - let active = policy_get(cx, sandbox).await?; - let revisions = policy_revisions(cx, sandbox).await?; + let active = policy_get(sandbox).await?; + let revisions = sandbox.policy_revisions().await?; let statuses = statuses(&revisions); testkit::ensure!( statuses.iter().all(|(_, status)| status != "failed"), diff --git a/tests/suites/conformance/src/port_forward.rs b/tests/suites/conformance/src/port_forward.rs index 999a7988a2..1bc05e7abd 100644 --- a/tests/suites/conformance/src/port_forward.rs +++ b/tests/suites/conformance/src/port_forward.rs @@ -7,10 +7,9 @@ use std::time::Duration; +use sandbox_clients_support::scripts::ECHO_SERVER; use testkit::{Context, policy}; -const ECHO_SERVER: &str = include_str!("scripts/echo_server.py"); - /// How long `forward start` and the first echo through it may take. const FORWARD_TIMEOUT: Duration = Duration::from_secs(60); diff --git a/tests/support/sandbox-clients/scripts/websocket_binary_client.py b/tests/support/sandbox-clients/scripts/websocket_binary_client.py new file mode 100644 index 0000000000..9ad9b28eb7 --- /dev/null +++ b/tests/support/sandbox-clients/scripts/websocket_binary_client.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Upgrades to WebSocket at argv[1]:argv[2]/ws, sends one binary frame, and +# waits up to three seconds for a reply or the connection to close. + +import base64 +import os +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + key = base64.b64encode(os.urandom(16)).decode() + sock.sendall( + f"GET /ws HTTP/1.1\r\nHost: {HOST}:{PORT}\r\nUpgrade: websocket\r\n" + f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n".encode() + ) + response = b"" + while b"\r\n\r\n" not in response: + chunk = sock.recv(4096) + if not chunk: + sys.exit(f"closed during the upgrade: {response!r}") + response += chunk + if not response.startswith(b"HTTP/1.1 101"): + sys.exit(f"upgrade failed: {response!r}") + payload, mask = b"binary-credential-channel", os.urandom(4) + sock.sendall( + bytes([0x82, 0x80 | len(payload)]) + + mask + + bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + ) + sock.settimeout(3) + try: + sock.recv(4096) + except (TimeoutError, ConnectionError): + pass diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs index 0be0298727..38a6093b9c 100644 --- a/tests/support/sandbox-clients/src/lib.rs +++ b/tests/support/sandbox-clients/src/lib.rs @@ -55,4 +55,58 @@ pub mod scripts { /// reply, and prints the outcome as one JSON line. [`crate::connect`] and /// [`crate::http`] run it and parse its output. pub const TCP_PROBE: &str = include_str!("../scripts/tcp_probe.py"); + + /// Serves HTTP on 127.0.0.1 and fetches from it, and prints whether the + /// environment has proxy variables and the payload it got. + pub const LOCALHOST_CLIENT: &str = include_str!("../scripts/localhost_client.py"); + + /// Closes listeners and exits a child that owns one, and fails unless the + /// port then refuses connections and can be bound again. Prints the child + /// listener's address and socket inode. + pub const LISTENER_LIFETIME_PROBE: &str = include_str!("../scripts/listener_lifetime_probe.py"); + + /// A sandbox command that opens an inheritable socket, writes its inode to + /// `/sandbox/preconnect-inode`, and sleeps. + pub const PRECONNECT_OWNER: &str = include_str!("../scripts/preconnect_owner.py"); + + /// Fails if the process holds the socket whose inode is its argument. + pub const SOCKET_NOT_INHERITED: &str = include_str!("../scripts/socket_not_inherited.py"); + + /// Holds a keep-alive connection to a host and port across a policy + /// reload, signalling through files in `/tmp`. + pub const RELOAD_KEEP_ALIVE_CLIENT: &str = + include_str!("../scripts/reload_keep_alive_client.py"); + + /// Sends GraphQL queries, mutations and persisted queries with urllib and + /// raw sockets, and prints each status. + pub const GRAPHQL_CLIENT: &str = include_str!("../scripts/graphql_client.py"); + + /// Sends JSON-RPC calls, batches and invalid JSON with urllib and raw + /// sockets, and prints each status. + pub const JSONRPC_CLIENT: &str = include_str!("../scripts/jsonrpc_client.py"); + + /// Sends an unlisted JSON-RPC method and a response frame, and prints each + /// status. + pub const JSONRPC_AUDIT_CLIENT: &str = include_str!("../scripts/jsonrpc_audit_client.py"); + + /// Runs an MCP client for each protocol revision it's given, and prints + /// `MCP_OK version=` after each passes. + pub const MCP_CLIENT: &str = include_str!("../scripts/mcp_client.py"); + + /// A TCP server on 127.0.0.1 at its argument's port that answers each + /// connection with `echo:` and the bytes it read. + pub const ECHO_SERVER: &str = include_str!("../scripts/echo_server.py"); + + /// Upgrades to WebSocket, sends one binary frame, and waits briefly for a + /// reply. + pub const WEBSOCKET_BINARY_CLIENT: &str = include_str!("../scripts/websocket_binary_client.py"); + + /// Upgrades to WebSocket, sends the token in `E2E_GATING_TOKEN` in one text + /// frame, and prints the server's text reply. + pub const WEBSOCKET_TEXT_CLIENT: &str = include_str!("../scripts/websocket_text_client.py"); + + /// A bash script, not Python: sends the token in `BOUND_TOKEN_A` to a + /// bound endpoint, another provider's endpoint and an unbound path, and + /// prints the first body and the other two statuses. + pub const BOUND_CREDENTIAL_CLIENT: &str = include_str!("../scripts/bound_credential_client.sh"); } From 0c73a5bd27b13e4b9d8b343b26ac4271a63c6516 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 22:32:33 -0800 Subject: [PATCH 28/63] docs(testkit): point the sandbox suffix note at MAX_SANDBOX_NAME_LEN Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/context.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index c7ab8839a1..cb80cd65be 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -173,8 +173,8 @@ impl Context { /// A resource name unique to this run: `test--`. /// - /// The gateway limits sandbox names to 19 characters, so a sandbox - /// suffix can have at most 5. + /// Sandbox names can't exceed [`MAX_SANDBOX_NAME_LEN`], so a sandbox + /// suffix can have at most 5 characters. pub fn name(&self, suffix: &str) -> String { format!("test-{}-{suffix}", self.run_id) } From 0bd55932ae2a320359d8ecf4d72c0ab74aedce95 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 10:30:49 -0800 Subject: [PATCH 29/63] test(conformance): cover the REST route and unlisted JSON-RPC methods Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/network_l7.rs | 12 ++++++++++-- .../sandbox-clients/scripts/jsonrpc_client.py | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs index 30c6845d3c..f48e961330 100644 --- a/tests/suites/conformance/src/network_l7.rs +++ b/tests/suites/conformance/src/network_l7.rs @@ -145,8 +145,9 @@ async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::R } /// JSON-RPC rules allow listed methods whatever their params, deny a -/// deny-rule method, a batch with one denied call, and invalid JSON, through -/// urllib and raw sockets. +/// deny-rule method, an unlisted method, a batch with one denied call, and +/// invalid JSON, through urllib and raw sockets. A REST endpoint on the same +/// port still allows requests outside the RPC path. #[testkit::test] async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Result { let server = HttpRecorder::serve(jsonrpc_upstream).await?; @@ -155,6 +156,11 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu jsonrpc: name: jsonrpc endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + enforcement: enforce + access: read-write - host: {TEST_HOST} port: {port} path: /rpc @@ -194,11 +200,13 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu ("method_tools_call_allowed", 200), ("method_tools_call_with_unmatched_params_allowed", 200), ("method_tools_delete_denied", 403), + ("unlisted_method_denied", 403), ] { expected.push((format!("{path}_{key}"), status)); } } for (key, status) in [ + ("rest_route_allowed", 200), ("forward_batch_all_allowed", 200), ("forward_batch_one_denied", 403), ("forward_invalid_json_denied", 403), diff --git a/tests/support/sandbox-clients/scripts/jsonrpc_client.py b/tests/support/sandbox-clients/scripts/jsonrpc_client.py index 919009584f..1d2b9b2608 100644 --- a/tests/support/sandbox-clients/scripts/jsonrpc_client.py +++ b/tests/support/sandbox-clients/scripts/jsonrpc_client.py @@ -162,6 +162,12 @@ def raw_jsonrpc_status(method, params, label): return raw_http_status(label, request) results = { + # The broad REST endpoint still permits requests outside the RPC route. + "rest_route_allowed": raw_http_status("rest_route_allowed", ( + f"POST /api/status HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + f"Content-Length: 0\r\nConnection: close\r\n\r\n" + ).encode()), # forward proxy — method-only allow rules "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}), "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), @@ -170,6 +176,7 @@ def raw_jsonrpc_status(method, params, label): "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {"name": "read_status"}), "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}), "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {"name": "purge_cache"}), + "forward_unlisted_method_denied": post_jsonrpc("forward_unlisted_method_denied", "admin.delete"), # forward proxy — batch: all requests allowed "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ @@ -192,6 +199,7 @@ def raw_jsonrpc_status(method, params, label): "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {"name": "read_status"}, "raw_method_tools_call_allowed"), "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}, "raw_method_tools_call_with_unmatched_params_allowed"), "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {"name": "purge_cache"}, "raw_method_tools_delete_denied"), + "raw_unlisted_method_denied": raw_jsonrpc_status("admin.delete", None, "raw_unlisted_method_denied"), } results.update(DETAILS) print(json.dumps(results, sort_keys=True)) From 644925f26d3cfaf60d3941aa77dd78719676aeba Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:46:45 -0800 Subject: [PATCH 30/63] test(conformance): assert documented error codes on L7 and pipelined denials Signed-off-by: Matthew Grossman --- .../suites/conformance/src/network_egress.rs | 16 +- tests/suites/conformance/src/network_l7.rs | 111 ++++--- .../sandbox-clients/scripts/graphql_client.py | 275 +++++------------- .../scripts/jsonrpc_audit_client.py | 15 +- .../sandbox-clients/scripts/jsonrpc_client.py | 232 ++++----------- tests/support/sandbox-clients/src/lib.rs | 6 +- 6 files changed, 224 insertions(+), 431 deletions(-) diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 85ff744caf..01376b2970 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -406,15 +406,25 @@ fn rest_policy(port: u16, method: &str) -> Result { } /// Send `requests` pipelined in one write, and fail unless the first gets -/// `200` and the second `403`. +/// `200` and the second a `403` with `policy_denied`. async fn run_pipeline(sandbox: &Sandbox, port: u16, requests: &str) -> Result { let reply = http_raw(sandbox, TEST_HOST, port, requests.as_bytes()).await?; let received = reply.received_text(); + let denial_error = received + .split_once("HTTP/1.1 403 Forbidden\r\n") + .and_then(|(_, denial)| denial.split_once("\r\n\r\n")) + .and_then(|(_, body)| { + serde_json::Deserializer::from_str(body) + .into_iter::() + .next()? + .ok() + }) + .and_then(|body| body.get("error")?.as_str().map(str::to_owned)); testkit::ensure!( reply.status() == Some(200) && received.matches("HTTP/1.1 ").count() == 2 - && received.contains("HTTP/1.1 403 Forbidden"), - "expected 200 then 403; got {reply:?}" + && denial_error.as_deref() == Some("policy_denied"), + "expected 200 then 403 with policy_denied; got {reply:?}" ); Ok(()) } diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs index f48e961330..7b8b219156 100644 --- a/tests/suites/conformance/src/network_l7.rs +++ b/tests/suites/conformance/src/network_l7.rs @@ -29,14 +29,17 @@ async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result .policy(rest_policy(server.port, "enforce")?) .create_ready() .await?; - for (request, expected) in [ - (clients::Request::get("/allowed"), 200), - (clients::Request::new("POST", "/allowed").body("test"), 403), + for (request, (status, error)) in [ + (clients::Request::get("/allowed"), ALLOWED), + ( + clients::Request::new("POST", "/allowed").body("test"), + POLICY_DENIED, + ), ] { let reply = http(&sandbox, TEST_HOST, server.port, &request).await?; testkit::ensure!( - reply.status() == Some(expected), - "expected {expected} for {request:?}; got {reply:?}" + reply.status() == Some(status) && error_code(&reply).as_deref() == error, + "expected {status} with error {error:?} for {request:?}; got {reply:?}" ); } let methods: Vec = server @@ -127,21 +130,21 @@ async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::R .await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { - for (key, status) in [ - ("query_allowed", 200), - ("get_query_allowed", 200), - ("duplicate_get_denied", 403), - ("persisted_get_allowed", 200), - ("unregistered_persisted_get_denied", 403), - ("chunked_query_allowed", 200), - ("unlisted_field_denied", 403), - ("mutation_allowed", 200), - ("deny_rule_denied", 403), + for (key, outcome) in [ + ("query_allowed", ALLOWED), + ("get_query_allowed", ALLOWED), + ("duplicate_get_denied", POLICY_DENIED), + ("persisted_get_allowed", ALLOWED), + ("unregistered_persisted_get_denied", POLICY_DENIED), + ("chunked_query_allowed", ALLOWED), + ("unlisted_field_denied", POLICY_DENIED), + ("mutation_allowed", ALLOWED), + ("deny_rule_denied", POLICY_DENIED), ] { - expected.push((format!("{path}_{key}"), status)); + expected.push((format!("{path}_{key}"), outcome)); } } - expect_statuses(&output, expected) + expect_outcomes(&output, expected) } /// JSON-RPC rules allow listed methods whatever their params, deny a @@ -194,26 +197,26 @@ async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Resu .await?; let mut expected = Vec::new(); for path in ["forward", "raw"] { - for (key, status) in [ - ("method_initialize_allowed", 200), - ("method_tools_list_allowed", 200), - ("method_tools_call_allowed", 200), - ("method_tools_call_with_unmatched_params_allowed", 200), - ("method_tools_delete_denied", 403), - ("unlisted_method_denied", 403), + for (key, outcome) in [ + ("method_initialize_allowed", ALLOWED), + ("method_tools_list_allowed", ALLOWED), + ("method_tools_call_allowed", ALLOWED), + ("method_tools_call_with_unmatched_params_allowed", ALLOWED), + ("method_tools_delete_denied", POLICY_DENIED), + ("unlisted_method_denied", POLICY_DENIED), ] { - expected.push((format!("{path}_{key}"), status)); + expected.push((format!("{path}_{key}"), outcome)); } } - for (key, status) in [ - ("rest_route_allowed", 200), - ("forward_batch_all_allowed", 200), - ("forward_batch_one_denied", 403), - ("forward_invalid_json_denied", 403), + for (key, outcome) in [ + ("rest_route_allowed", ALLOWED), + ("forward_batch_all_allowed", ALLOWED), + ("forward_batch_one_denied", POLICY_DENIED), + ("forward_invalid_json_denied", POLICY_DENIED), ] { - expected.push((key.to_string(), status)); + expected.push((key.to_string(), outcome)); } - expect_statuses(&output, expected) + expect_outcomes(&output, expected) } /// A JSON-RPC endpoint without `enforcement` audits an unlisted method and @@ -252,11 +255,14 @@ async fn jsonrpc_default_audit_mode_still_denies_response_frames( let output = sandbox .run_python(scripts::JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]) .await?; - expect_statuses( + expect_outcomes( &output, [ - ("forward_unknown_method_audited".to_string(), 200), - ("forward_response_frame_hard_denied".to_string(), 403), + ("forward_unknown_method_audited".to_string(), ALLOWED), + ( + "forward_response_frame_hard_denied".to_string(), + POLICY_DENIED, + ), ], ) } @@ -317,36 +323,55 @@ async fn expect_upgrade_refused(sandbox: &Sandbox, port: u16) -> Result { Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" ); let reply = http_raw(sandbox, TEST_HOST, port, request.as_bytes()).await?; - let received = reply.received_text(); testkit::ensure!( reply.status() == Some(403) - && received.contains("unsupported_l7_protocol") - && received.contains("REST endpoints"), + && error_code(&reply).as_deref() == Some("unsupported_l7_protocol") + && reply.received_text().contains("REST endpoints"), "expected the upgrade to {target} refused; got {reply:?}" ); } Ok(()) } +/// A response's status and the `error` code in its JSON body, if any. +type Outcome = (u16, Option<&'static str>); + +/// Upstream answered. +const ALLOWED: Outcome = (200, None); + +/// The sandbox denied the request by policy, as documented under "Proxy +/// Error Responses" in `docs/observability/logging.mdx`. +const POLICY_DENIED: Outcome = (403, Some("policy_denied")); + /// Fail unless the client's JSON result maps exactly the keys in -/// `expected` to their statuses. -fn expect_statuses( +/// `expected` to their statuses and `error` codes. +fn expect_outcomes( output: &testkit::ExecOutput, - expected: impl IntoIterator, + expected: impl IntoIterator, ) -> Result { let result = output.last_json()?; let expected: Value = expected .into_iter() - .map(|(key, status)| (key, json!(status))) + .map(|(key, (status, error))| (key, json!({"status": status, "error": error}))) .collect::>() .into(); testkit::ensure!( result == expected, - "unexpected statuses:\nexpected: {expected}\ngot: {result}" + "unexpected outcomes:\nexpected: {expected}\ngot: {result}" ); Ok(()) } +/// The `error` code in the JSON body of `reply`'s response, if any. +pub(crate) fn error_code(reply: &clients::Http) -> Option { + let body = &reply.response.as_ref()?.body; + let error = serde_json::from_slice::(body) + .ok()? + .get("error")? + .clone(); + error.as_str().map(str::to_owned) +} + fn ok_json() -> Response { Response::json(&json!({"ok": true})) } diff --git a/tests/support/sandbox-clients/scripts/graphql_client.py b/tests/support/sandbox-clients/scripts/graphql_client.py index f8d4c07c6a..636f5dcb83 100644 --- a/tests/support/sandbox-clients/scripts/graphql_client.py +++ b/tests/support/sandbox-clients/scripts/graphql_client.py @@ -2,241 +2,104 @@ # SPDX-License-Identifier: Apache-2.0 # Sends GraphQL queries, mutations, and persisted queries to HOST:PORT with -# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as -# JSON. +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status and +# the `error` code of its JSON body as JSON. import json import socket import sys -import time import urllib.error import urllib.parse import urllib.request HOST = sys.argv[1] PORT = int(sys.argv[2]) -DETAILS = {} +TARGET = f"{HOST}:{PORT}" QUERY_VIEWER = "query Viewer { viewer { login } }" QUERY_REPOSITORY = "query Repo { repository(owner:\"o\", name:\"r\") { id } }" MUTATION_CREATE = "mutation Create { createIssue(input:{repositoryId:\"r\", title:\"t\", body:\"b\"}) { issue { id } } }" MUTATION_DELETE = "mutation Delete { deleteRepository(input:{repositoryId:\"r\"}) { clientMutationId } }" -def forward_status(query): - body = json.dumps({"query": query}).encode() - request = urllib.request.Request( - f"http://{HOST}:{PORT}/graphql", - data=body, - headers={"Content-Type": "application/json"}, - method="POST", - ) +def outcome(status, body): try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + return {"status": status, "error": error if isinstance(error, str) else None} -def forward_get_status(query): - encoded = urllib.parse.urlencode({"query": query}) - request = urllib.request.Request( - f"http://{HOST}:{PORT}/graphql?{encoded}", - method="GET", - ) +def urllib_outcome(request): try: with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status + return outcome(response.status, response.read()) except urllib.error.HTTPError as error: - error.read() - return error.code + return outcome(error.code, error.read()) -def forward_duplicate_get_status(): - safe = urllib.parse.quote_plus(QUERY_VIEWER) - unsafe = urllib.parse.quote_plus(MUTATION_DELETE) - request = urllib.request.Request( - f"http://{HOST}:{PORT}/graphql?query={safe}&query={unsafe}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code +def forward_post(query): + return urllib_outcome(urllib.request.Request( + f"http://{TARGET}/graphql", + data=json.dumps({"query": query}).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + )) -def forward_persisted_get_status(hash_value): - extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) - encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) - request = urllib.request.Request( - f"http://{HOST}:{PORT}/graphql?{encoded}", +def forward_get(query_string): + return urllib_outcome(urllib.request.Request( + f"http://{TARGET}/graphql?{query_string}", method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_chunked_status(query): - target = f"{HOST}:{PORT}" - body = json.dumps({"query": query}).encode() - chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + )) +def raw(request): with socket.create_connection((HOST, PORT), timeout=15) as sock: - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk sock.sendall(request) - response, body = read_response(sock) - DETAILS["forward_chunked_query_allowed_detail"] = body.decode(errors="replace") - return status_code(response, "forward_chunked_response") - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{label}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{label}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response = read_until(sock, b"\r\n\r\n") - return status_code(response, f"{label}_response") - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") - -def raw_status(query, label): - target = f"{HOST}:{PORT}" + response = b"" + while chunk := sock.recv(4096): + response += chunk + head, _, body = response.partition(b"\r\n\r\n") + status_line = head.split(b"\r\n", 1)[0].split() + if len(status_line) < 2 or not status_line[1].isdigit(): + return {"status": None, "error": None, "raw": response.decode(errors="replace")} + return outcome(int(status_line[1]), body) + +def raw_post(query, chunked=False): body = json.dumps({"query": query}).encode() - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {len(body)}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + body - return raw_http_status(label, request) - -def raw_get_status(query, label): - target = f"{HOST}:{PORT}" - encoded = urllib.parse.urlencode({"query": query}) - - request = ( - f"GET /graphql?{encoded} HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_duplicate_get_status(): - target = f"{HOST}:{PORT}" + if chunked: + framing = "Transfer-Encoding: chunked" + body = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + else: + framing = f"Content-Length: {len(body)}" + return raw(( + f"POST /graphql HTTP/1.1\r\nHost: {TARGET}\r\n" + f"Content-Type: application/json\r\n{framing}\r\nConnection: close\r\n\r\n" + ).encode() + body) + +def raw_get(query_string): + return raw(( + f"GET /graphql?{query_string} HTTP/1.1\r\nHost: {TARGET}\r\nConnection: close\r\n\r\n" + ).encode()) + +def get_query(query): + return urllib.parse.urlencode({"query": query}) + +def duplicate_get_query(): safe = urllib.parse.quote_plus(QUERY_VIEWER) unsafe = urllib.parse.quote_plus(MUTATION_DELETE) + return f"query={safe}&query={unsafe}" - request = ( - f"GET /graphql?query={safe}&query={unsafe} HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status("raw_duplicate_get_denied", request) - -def raw_persisted_get_status(hash_value, label): - target = f"{HOST}:{PORT}" +def persisted_get_query(hash_value): extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) - encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) - - request = ( - f"GET /graphql?{encoded} HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_chunked_status(query): - target = f"{HOST}:{PORT}" - body = json.dumps({"query": query}).encode() - chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk - return raw_http_status("raw_chunked_query_allowed", request) - -results = { - "forward_query_allowed": forward_status(QUERY_VIEWER), - "forward_get_query_allowed": forward_get_status(QUERY_VIEWER), - "forward_duplicate_get_denied": forward_duplicate_get_status(), - "forward_persisted_get_allowed": forward_persisted_get_status("abc123"), - "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"), - "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER), - "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), - "forward_mutation_allowed": forward_status(MUTATION_CREATE), - "forward_deny_rule_denied": forward_status(MUTATION_DELETE), - "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), - "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), - "raw_duplicate_get_denied": raw_duplicate_get_status(), - "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), - "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), - "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), - "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), - "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), - "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), -} -results.update(DETAILS) + return urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) + +results = {} +for path, post, get in [("forward", forward_post, forward_get), ("raw", raw_post, raw_get)]: + results.update({ + f"{path}_query_allowed": post(QUERY_VIEWER), + f"{path}_get_query_allowed": get(get_query(QUERY_VIEWER)), + f"{path}_duplicate_get_denied": get(duplicate_get_query()), + f"{path}_persisted_get_allowed": get(persisted_get_query("abc123")), + f"{path}_unregistered_persisted_get_denied": get(persisted_get_query("missing")), + f"{path}_chunked_query_allowed": raw_post(QUERY_VIEWER, chunked=True), + f"{path}_unlisted_field_denied": post(QUERY_REPOSITORY), + f"{path}_mutation_allowed": post(MUTATION_CREATE), + f"{path}_deny_rule_denied": post(MUTATION_DELETE), + }) print(json.dumps(results, sort_keys=True)) diff --git a/tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py b/tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py index c1145be040..1d5a53a70a 100644 --- a/tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py +++ b/tests/support/sandbox-clients/scripts/jsonrpc_audit_client.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # Sends an unlisted JSON-RPC method and a response frame to HOST:PORT/rpc, -# and prints each status as JSON. +# and prints each status and the `error` code of its JSON body as JSON. import json import sys import urllib.error @@ -11,6 +11,13 @@ HOST = sys.argv[1] PORT = int(sys.argv[2]) +def outcome(status, body): + try: + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + return {"status": status, "error": error if isinstance(error, str) else None} + def post_jsonrpc(body): encoded = json.dumps(body).encode() request = urllib.request.Request( @@ -21,11 +28,9 @@ def post_jsonrpc(body): ) try: with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status + return outcome(response.status, response.read()) except urllib.error.HTTPError as error: - error.read() - return error.code + return outcome(error.code, error.read()) results = { "forward_unknown_method_audited": post_jsonrpc({"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}), diff --git a/tests/support/sandbox-clients/scripts/jsonrpc_client.py b/tests/support/sandbox-clients/scripts/jsonrpc_client.py index 1d2b9b2608..1e4cc01a54 100644 --- a/tests/support/sandbox-clients/scripts/jsonrpc_client.py +++ b/tests/support/sandbox-clients/scripts/jsonrpc_client.py @@ -2,204 +2,94 @@ # SPDX-License-Identifier: Apache-2.0 # Sends JSON-RPC calls, batches, and invalid JSON to HOST:PORT/rpc with -# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as -# JSON. +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status and +# the `error` code of its JSON body as JSON. import json import socket import sys -import time import urllib.error import urllib.request HOST = sys.argv[1] PORT = int(sys.argv[2]) -DETAILS = { - "debug_target": {"host": HOST, "port": PORT}, -} - -def text(data): - return data.decode(errors="replace") - -def selected_headers(headers): - return { - key.lower(): value - for key, value in headers.items() - if key.lower() in ("content-type", "content-length", "server") - } +TARGET = f"{HOST}:{PORT}" -def record_http_error(label, error, request_body): - response_body = error.read() - DETAILS[f"{label}_request"] = request_body - DETAILS[f"{label}_response"] = { - "status": error.code, - "reason": str(error.reason), - "headers": selected_headers(error.headers), - "body": text(response_body), - } - return error.code - -def post_jsonrpc(label, method, params=None, req_id=1): - body = {"jsonrpc": "2.0", "id": req_id, "method": method} - if params is not None: - body["params"] = params - encoded = json.dumps(body).encode() - request = urllib.request.Request( - f"http://{HOST}:{PORT}/rpc", - data=encoded, - headers={"Content-Type": "application/json"}, - method="POST", - ) +def outcome(status, body): try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, body) + error = json.loads(body).get("error") + except (ValueError, AttributeError): + error = None + return {"status": status, "error": error if isinstance(error, str) else None} -def post_jsonrpc_batch(label, requests): - encoded = json.dumps(requests).encode() +def forward(body): request = urllib.request.Request( - f"http://{HOST}:{PORT}/rpc", - data=encoded, + f"http://{TARGET}/rpc", + data=body, headers={"Content-Type": "application/json"}, method="POST", ) try: with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status + return outcome(response.status, response.read()) except urllib.error.HTTPError as error: - return record_http_error(label, error, requests) - -def post_invalid_json(label): - encoded = b"not valid json {" - request = urllib.request.Request( - f"http://{HOST}:{PORT}/rpc", - data=encoded, - headers={"Content-Type": "application/json", "Content-Length": str(len(encoded))}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, text(encoded)) - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{label}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{label}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error - -def record_raw_response(label, response, body=b""): - code = status_code(response, label) - if code != 200: - DETAILS[f"{label}_raw"] = text(response) - if body: - DETAILS[f"{label}_body"] = text(body) - return code - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response, body = read_response(sock) - return record_raw_response(f"{label}_response", response, body) - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") - -def raw_jsonrpc_status(method, params, label): - target = f"{HOST}:{PORT}" + return outcome(error.code, error.read()) + +def raw(request): + with socket.create_connection((HOST, PORT), timeout=15) as sock: + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response = b"" + while chunk := sock.recv(4096): + response += chunk + head, _, body = response.partition(b"\r\n\r\n") + status_line = head.split(b"\r\n", 1)[0].split() + if len(status_line) < 2 or not status_line[1].isdigit(): + return {"status": None, "error": None, "raw": response.decode(errors="replace")} + return outcome(int(status_line[1]), body) + +def call(method, params=None): body = {"jsonrpc": "2.0", "id": 1, "method": method} if params is not None: body["params"] = params - encoded = json.dumps(body).encode() - request = ( - f"POST /rpc HTTP/1.1\r\n" - f"Host: {target}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {len(encoded)}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + encoded - return raw_http_status(label, request) + return json.dumps(body).encode() + +def raw_post(path, body): + return raw(( + f"POST {path} HTTP/1.1\r\nHost: {TARGET}\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n" + ).encode() + body) + +def raw_rpc(body): + return raw_post("/rpc", body) + +INITIALIZE = call("initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}) +CALLS = { + "method_initialize_allowed": INITIALIZE, + "method_tools_list_allowed": call("tools/list"), + "method_tools_call_allowed": call("tools/call", {"name": "read_status"}), + "method_tools_call_with_unmatched_params_allowed": call( + "tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}} + ), + "method_tools_delete_denied": call("tools/delete", {"name": "purge_cache"}), + "unlisted_method_denied": call("admin.delete"), +} results = { # The broad REST endpoint still permits requests outside the RPC route. - "rest_route_allowed": raw_http_status("rest_route_allowed", ( - f"POST /api/status HTTP/1.1\r\n" - f"Host: {HOST}:{PORT}\r\n" - f"Content-Length: 0\r\nConnection: close\r\n\r\n" - ).encode()), - # forward proxy — method-only allow rules - "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}), - "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), - - # forward proxy — method allow/deny rules - "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {"name": "read_status"}), - "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}), - "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {"name": "purge_cache"}), - "forward_unlisted_method_denied": post_jsonrpc("forward_unlisted_method_denied", "admin.delete"), - - # forward proxy — batch: all requests allowed - "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ + "rest_route_allowed": raw_post("/api/status", b""), + # One denied call in a batch denies the whole batch. + "forward_batch_all_allowed": forward(json.dumps([ {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "read_status"}}, - ]), - - # forward proxy — batch: one denied request causes full batch denial - "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ + ]).encode()), + "forward_batch_one_denied": forward(json.dumps([ {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, {"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {"name": "purge_cache"}}, - ]), - - # forward proxy — invalid JSON body fails closed before generic rules apply - "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), - - # raw socket path — representative allowed and denied cases - "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}, "raw_method_initialize_allowed"), - "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), - "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {"name": "read_status"}, "raw_method_tools_call_allowed"), - "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}, "raw_method_tools_call_with_unmatched_params_allowed"), - "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {"name": "purge_cache"}, "raw_method_tools_delete_denied"), - "raw_unlisted_method_denied": raw_jsonrpc_status("admin.delete", None, "raw_unlisted_method_denied"), + ]).encode()), + # An invalid JSON body fails closed before generic rules apply. + "forward_invalid_json_denied": forward(b"not valid json {"), } -results.update(DETAILS) +for key, body in CALLS.items(): + results[f"forward_{key}"] = forward(body) + results[f"raw_{key}"] = raw_rpc(body) print(json.dumps(results, sort_keys=True)) diff --git a/tests/support/sandbox-clients/src/lib.rs b/tests/support/sandbox-clients/src/lib.rs index 38a6093b9c..959d4ffb3b 100644 --- a/tests/support/sandbox-clients/src/lib.rs +++ b/tests/support/sandbox-clients/src/lib.rs @@ -78,15 +78,15 @@ pub mod scripts { include_str!("../scripts/reload_keep_alive_client.py"); /// Sends GraphQL queries, mutations and persisted queries with urllib and - /// raw sockets, and prints each status. + /// raw sockets, and prints each status and JSON `error` code. pub const GRAPHQL_CLIENT: &str = include_str!("../scripts/graphql_client.py"); /// Sends JSON-RPC calls, batches and invalid JSON with urllib and raw - /// sockets, and prints each status. + /// sockets, and prints each status and JSON `error` code. pub const JSONRPC_CLIENT: &str = include_str!("../scripts/jsonrpc_client.py"); /// Sends an unlisted JSON-RPC method and a response frame, and prints each - /// status. + /// status and JSON `error` code. pub const JSONRPC_AUDIT_CLIENT: &str = include_str!("../scripts/jsonrpc_audit_client.py"); /// Runs an MCP client for each protocol revision it's given, and prints From 319e28ef6bcdc0ea80d856e9d6f4702f002eb36e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 15:46:50 -0800 Subject: [PATCH 31/63] test(conformance): require the fail-closed middleware's 403 and match the finding name Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/network_egress.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 01376b2970..a254fe02cf 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -230,28 +230,14 @@ async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) - let mut payload = vec![0x00, 0xff, 0x13, 0x37]; payload.extend_from_slice(b"not-http-or-tls"); let reply = http_raw(&sandbox, TEST_HOST, server.port, &payload).await?; - let denial = reply.received_text(); testkit::ensure!( - reply.connect.is_connected() - && (denial.is_empty() - || denial.contains("HTTP/1.1 403 Forbidden") - && denial.contains("unsupported_l7_protocol")), + reply.status() == Some(403) + && crate::network_l7::error_code(&reply).as_deref() == Some("unsupported_l7_protocol"), "expected the middleware to block the payload: {reply:?}" ); - // Give a forwarded payload time to arrive before checking it didn't. - tokio::time::sleep(Duration::from_millis(200)).await; - let received = server.received(); - testkit::ensure!( - received.is_empty(), - "the payload reached upstream: {:?}", - String::from_utf8_lossy(&received) - ); sandbox .logs_until("the middleware denial", |logs| { logs.contains("openshell.middleware.traffic_uninspectable") - && logs.contains( - "Unsupported tunnel protocol cannot be inspected by required middleware", - ) }) .await?; Ok(()) From 881cee83d1d2ad5a55ed6d1aed27d362967fe4c4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 21:11:03 -0800 Subject: [PATCH 32/63] test(conformance): skip sandbox log checks on multi-replica gateways Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 3 +++ tests/suites/conformance/src/lib.rs | 22 +++++++++++++++++++ .../suites/conformance/src/network_egress.rs | 3 +++ .../ansible/playbooks/openshell-k3s-ha.yaml | 9 ++++++++ 4 files changed, 37 insertions(+) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 7ac6a43983..9d9b167682 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -383,6 +383,9 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: output.stdout.replace(SECRET_A, "") ); + if crate::sandbox_logs_unreliable("profile_endpoints_bind_static_credentials") { + return Ok(()); + } let logs = sandbox .logs_until("the endpoint mismatch", |logs| { logs.contains("openshell.provider_credential.endpoint_mismatch") diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index e3a4726e9e..791f24db20 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -31,3 +31,25 @@ mod smoke; mod ssh; mod templates; mod workspaces; + +/// Names the number of gateway replicas when an environment runs more than +/// one. +const GATEWAY_REPLICAS_ENV: &str = "OPENSHELL_TEST_GATEWAY_REPLICAS"; + +/// Whether sandbox log reads can miss a sandbox's logs: a gateway replica +/// holds only the logs of the sandboxes connected to it, and a read can reach +/// another one (#5462). Says so when they can, so the test can skip its log +/// check. +fn sandbox_logs_unreliable(test: &str) -> bool { + let replicas = std::env::var(GATEWAY_REPLICAS_ENV) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(1); + if replicas > 1 { + eprintln!( + "[{test}] skipping the log check: the gateway has {replicas} replicas and \ + sandbox logs are read from only one (#5462)" + ); + } + replicas > 1 +} diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index a254fe02cf..1324834777 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -235,6 +235,9 @@ async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) - && crate::network_l7::error_code(&reply).as_deref() == Some("unsupported_l7_protocol"), "expected the middleware to block the payload: {reply:?}" ); + if crate::sandbox_logs_unreliable("fail_closed_middleware_blocks_uninspectable_traffic") { + return Ok(()); + } sandbox .logs_until("the middleware denial", |logs| { logs.contains("openshell.middleware.traffic_uninspectable") diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml index efcc736d49..bb25995fd8 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml @@ -104,3 +104,12 @@ become: false vars: openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" + + # Sandbox logs are read from one replica only (#5462); tests that read + # them skip those checks here. + - name: Publish gateway test variables + ansible.builtin.include_tasks: test-env.yaml + vars: + test_env_name: gateway + test_env_vars: + OPENSHELL_TEST_GATEWAY_REPLICAS: "3" From b7b28644a56d8bcbc4731521484f857b7af22a0c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 21:17:44 -0800 Subject: [PATCH 33/63] test(support): record the address each host server request arrived at Signed-off-by: Matthew Grossman --- tests/support/host-server/src/http.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index 5aec7932ff..b93e6dddcc 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use std::fmt::Write as _; +use std::net::SocketAddr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; @@ -18,6 +19,9 @@ pub struct Request { /// The request line and headers. pub head: String, pub body: String, + /// The server's address on the connection the request arrived on: the + /// address of the test host the client reached it by. + pub local_addr: Option, } impl Request { @@ -260,8 +264,9 @@ async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc) -> Option { let mut chunk = [0_u8; 4096]; loop { - if let Some((request, length)) = parse_request(buffer) { + if let Some((mut request, length)) = parse_request(buffer) { buffer.drain(..length); + request.local_addr = stream.local_addr().ok(); return Some(request); } match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { @@ -289,7 +294,14 @@ fn parse_request(bytes: &[u8]) -> Option<(Request, usize)> { (rest.get(..length)?.to_vec(), length) }; let body = String::from_utf8_lossy(&body).into_owned(); - Some((Request { head, body }, end + 4 + length)) + Some(( + Request { + head, + body, + local_addr: None, + }, + end + 4 + length, + )) } /// A complete chunked body at the start of `bytes`, decoded, and its encoded From e6adae8dd586e518c9ed25a4a9a1ff4b638aab64 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 21:17:44 -0800 Subject: [PATCH 34/63] test(features): find the host address from the host server in routable-host-ip Signed-off-by: Matthew Grossman --- TESTING.md | 9 +-- .../tests/routable_host_ip.rs | 55 ++++++++++++++----- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/TESTING.md b/TESTING.md index b9ae46db6f..e234086747 100644 --- a/TESTING.md +++ b/TESTING.md @@ -657,10 +657,11 @@ the node's address. The `routable-host-ip` feature suite (`tests/suites/features/routable-host-ip`) checks that a policy endpoint whose host is an IP literal reaches that address, -with or without `allowed_ips`. The sandbox uses the address -`host.openshell.internal` resolves to, which must be one the supervisor lets an -IP literal reach. Rootless Podman's is link-local and always blocked, so CI runs -the suite on Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` +with or without `allowed_ips`. The test reaches a host server by +`host.openshell.internal`, then uses the address that request arrived at, which +must be one the supervisor lets an IP literal reach. Rootless Podman's is +link-local and always blocked, and Docker Desktop's is loopback, so CI runs the +suite on Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` installers. The `connect-ssh-supervision` feature suite diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs index 15c2720768..b05e07a664 100644 --- a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -4,8 +4,10 @@ //! A policy endpoint whose host is an IP literal reaches that address //! transparently, whether `allowed_ips` names it or not. //! -//! The servers run on the test host, and the sandbox reaches them by the -//! address `host.openshell.internal` resolves to. Not conformance, because +//! The servers run on the test host. The sandbox can't learn the host's +//! address, since policy DNS answers it with a synthetic one, so the test +//! reaches a server by `host.openshell.internal` first and takes the address +//! the server saw the request arrive at. Not conformance, because //! that address must be one the supervisor lets an IP literal reach: rootless //! Podman's is link-local, which is always blocked. //! @@ -15,7 +17,7 @@ use host_server_support::{HttpRecorder, TEST_HOST}; use sandbox_clients_support::{Request, http}; use testkit::policy; -use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; +use testkit::{Context, PYTHON_BINARIES}; /// Requests reach a server by an IP literal endpoint with `allowed_ips` for /// that address, and another by an IP literal endpoint without it. @@ -23,23 +25,46 @@ use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Result { let explicit = HttpRecorder::start("explicit").await?; let implicit = HttpRecorder::start("implicit").await?; + let mut name_policy = policy::base(); + policy::allow( + &mut name_policy, + "by_name", + TEST_HOST, + explicit.port, + PYTHON_BINARIES, + ); let sandbox = cx .sandbox("hstip") - .policy(policy::base()) + .policy(name_policy) .image(testkit::python_image()) .command(["sleep", "infinity"]) .create_ready() .await?; - let ip = sandbox - .run_python( - "import socket, sys; print(socket.gethostbyname(sys.argv[1]))", - &[TEST_HOST], - ) - .await - .context(format!("resolve {TEST_HOST} in the sandbox"))? - .stdout - .trim() - .to_string(); + let reply = http( + &sandbox, + TEST_HOST, + explicit.port, + &Request::get("/by-name"), + ) + .await?; + testkit::ensure!( + reply.status() == Some(200), + "request to {TEST_HOST}:{} failed: {reply:?}", + explicit.port + ); + let host_ip = explicit + .requests() + .first() + .and_then(|request| request.local_addr) + .ok_or_else(|| testkit::Error::msg("the host server didn't record its address"))? + .ip(); + // Docker Desktop relays the host gateway through a host-side proxy. + testkit::ensure!( + !host_ip.is_loopback(), + "requests to {TEST_HOST} arrive at the host on {host_ip}, which a sandbox can't reach" + ); + let ip = host_ip.to_string(); + let prefix = if host_ip.is_ipv4() { 32 } else { 128 }; let mut ip_policy = policy::base(); policy::allow( @@ -49,7 +74,7 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul explicit.port, PYTHON_BINARIES, ) - .allowed_ips = vec![format!("{ip}/32")]; + .allowed_ips = vec![format!("{ip}/{prefix}")]; policy::allow( &mut ip_policy, "implicit", From 3bc5d6945f505425b0b1024a77efde678e76d646 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 22:22:23 -0800 Subject: [PATCH 35/63] test(conformance): skip the sandbox log tests entirely on multi-replica gateways Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/credential_gating.rs | 6 +++--- tests/suites/conformance/src/lib.rs | 14 +++++++------- tests/suites/conformance/src/network_egress.rs | 6 +++--- .../ansible/playbooks/openshell-k3s-ha.yaml | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 9d9b167682..53f7813080 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -326,6 +326,9 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te /// or its variable name. #[testkit::test] async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit::Result { + if crate::skip_reading_sandbox_logs("profile_endpoints_bind_static_credentials") { + return Ok(()); + } const SECRET_A: &str = "e2e-bound-secret"; const SECRET_B: &str = "e2e-provider-b-secret"; let authorized = |request: &Request| { @@ -383,9 +386,6 @@ async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit: output.stdout.replace(SECRET_A, "") ); - if crate::sandbox_logs_unreliable("profile_endpoints_bind_static_credentials") { - return Ok(()); - } let logs = sandbox .logs_until("the endpoint mismatch", |logs| { logs.contains("openshell.provider_credential.endpoint_mismatch") diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 791f24db20..1a0f0e66f7 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -36,19 +36,19 @@ mod workspaces; /// one. const GATEWAY_REPLICAS_ENV: &str = "OPENSHELL_TEST_GATEWAY_REPLICAS"; -/// Whether sandbox log reads can miss a sandbox's logs: a gateway replica -/// holds only the logs of the sandboxes connected to it, and a read can reach -/// another one (#5462). Says so when they can, so the test can skip its log -/// check. -fn sandbox_logs_unreliable(test: &str) -> bool { +/// Whether a test that reads sandbox logs should skip itself, and says so +/// when it should: a gateway replica holds only the logs of the sandboxes +/// connected to it, and a read can reach another one (#5462). Remove this +/// once #5462 is fixed or an environment can skip tests itself (#5752). +fn skip_reading_sandbox_logs(test: &str) -> bool { let replicas = std::env::var(GATEWAY_REPLICAS_ENV) .ok() .and_then(|value| value.parse::().ok()) .unwrap_or(1); if replicas > 1 { eprintln!( - "[{test}] skipping the log check: the gateway has {replicas} replicas and \ - sandbox logs are read from only one (#5462)" + "[{test}] skipping: the gateway has {replicas} replicas and sandbox logs \ + are read from only one (#5462, #5752)" ); } replicas > 1 diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 1324834777..1260e28983 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -218,6 +218,9 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result /// byte reaches upstream, and the sandbox logs why. #[testkit::test] async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { + if crate::skip_reading_sandbox_logs("fail_closed_middleware_blocks_uninspectable_traffic") { + return Ok(()); + } let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_closed", "")?; let sandbox = cx @@ -235,9 +238,6 @@ async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) - && crate::network_l7::error_code(&reply).as_deref() == Some("unsupported_l7_protocol"), "expected the middleware to block the payload: {reply:?}" ); - if crate::sandbox_logs_unreliable("fail_closed_middleware_blocks_uninspectable_traffic") { - return Ok(()); - } sandbox .logs_until("the middleware denial", |logs| { logs.contains("openshell.middleware.traffic_uninspectable") diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml index bb25995fd8..21d970497f 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml @@ -106,7 +106,7 @@ openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" # Sandbox logs are read from one replica only (#5462); tests that read - # them skip those checks here. + # them skip themselves here until #5752. - name: Publish gateway test variables ansible.builtin.include_tasks: test-env.yaml vars: From 86a2569fce52001bed9dacc0b082ae0ad36bdde7 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 11:38:04 -0800 Subject: [PATCH 36/63] test(conformance): mark the sandbox log tests unsupported on k3s-ha Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 3 --- tests/suites/conformance/src/lib.rs | 22 ------------------- .../suites/conformance/src/network_egress.rs | 3 --- tests/suites/conformance/unsupported.yaml | 11 +++++++++- .../ansible/playbooks/openshell-k3s-ha.yaml | 9 -------- 5 files changed, 10 insertions(+), 38 deletions(-) diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 53f7813080..7ac6a43983 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -326,9 +326,6 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te /// or its variable name. #[testkit::test] async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit::Result { - if crate::skip_reading_sandbox_logs("profile_endpoints_bind_static_credentials") { - return Ok(()); - } const SECRET_A: &str = "e2e-bound-secret"; const SECRET_B: &str = "e2e-provider-b-secret"; let authorized = |request: &Request| { diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 1a0f0e66f7..e3a4726e9e 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -31,25 +31,3 @@ mod smoke; mod ssh; mod templates; mod workspaces; - -/// Names the number of gateway replicas when an environment runs more than -/// one. -const GATEWAY_REPLICAS_ENV: &str = "OPENSHELL_TEST_GATEWAY_REPLICAS"; - -/// Whether a test that reads sandbox logs should skip itself, and says so -/// when it should: a gateway replica holds only the logs of the sandboxes -/// connected to it, and a read can reach another one (#5462). Remove this -/// once #5462 is fixed or an environment can skip tests itself (#5752). -fn skip_reading_sandbox_logs(test: &str) -> bool { - let replicas = std::env::var(GATEWAY_REPLICAS_ENV) - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(1); - if replicas > 1 { - eprintln!( - "[{test}] skipping: the gateway has {replicas} replicas and sandbox logs \ - are read from only one (#5462, #5752)" - ); - } - replicas > 1 -} diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 1260e28983..a254fe02cf 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -218,9 +218,6 @@ async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result /// byte reaches upstream, and the sandbox logs why. #[testkit::test] async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { - if crate::skip_reading_sandbox_logs("fail_closed_middleware_blocks_uninspectable_traffic") { - return Ok(()); - } let server = TcpServer::echo().await?; let policy = middleware_policy(server.port, "fail_closed", "")?; let sandbox = cx diff --git a/tests/suites/conformance/unsupported.yaml b/tests/suites/conformance/unsupported.yaml index e24d99290b..135f89720a 100644 --- a/tests/suites/conformance/unsupported.yaml +++ b/tests/suites/conformance/unsupported.yaml @@ -17,4 +17,13 @@ # the bug. A test can appear more than once with different issues, but not twice # for one environment. Branch Checks runs the program with each environment's # entries to validate this file; see tests/suites/conformance/README.md. -[] +# A gateway replica holds only the logs of the sandboxes connected to it, so a +# log read on k3s-ha can miss them (#5462). +- test: conformance::credential_gating::profile_endpoints_bind_static_credentials + issue: https://github.com/NVIDIA/OpenShell/issues/5752 + environments: + - ubuntu-k3s/k3s-ha +- test: conformance::network_egress::fail_closed_middleware_blocks_uninspectable_traffic + issue: https://github.com/NVIDIA/OpenShell/issues/5752 + environments: + - ubuntu-k3s/k3s-ha diff --git a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml index 21d970497f..efcc736d49 100644 --- a/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml +++ b/tests/tmachine/ansible/playbooks/openshell-k3s-ha.yaml @@ -104,12 +104,3 @@ become: false vars: openshell_client_gateway_endpoint: "https://openshell.openshell.svc.cluster.local:{{ k3s_gateway_address.stdout.split(':')[1] }}" - - # Sandbox logs are read from one replica only (#5462); tests that read - # them skip themselves here until #5752. - - name: Publish gateway test variables - ansible.builtin.include_tasks: test-env.yaml - vars: - test_env_name: gateway - test_env_vars: - OPENSHELL_TEST_GATEWAY_REPLICAS: "3" From 9959030f832460eaca76dd63f7bc7dc2d8a6266a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 37/63] docs(testkit): add the tests/support environment contract table Signed-off-by: Matthew Grossman --- tests/support/README.md | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/support/README.md diff --git a/tests/support/README.md b/tests/support/README.md new file mode 100644 index 0000000000..f2ca63d396 --- /dev/null +++ b/tests/support/README.md @@ -0,0 +1,42 @@ +# Test support crates + +Each crate here is a fixture for one concern, such as a service the +environment runs, and any suite can depend on it by path. See +[Adding a feature suite](../../TESTING.md#adding-a-feature-suite) for how an +environment and a fixture share a contract. + +## Environment contract + +These are the variables that an environment, the test runner or CI sets for +tests to read. A tmachine environment publishes its variables with +`tests/tmachine/ansible/playbooks/test-env.yaml`, and `openshell-test-runner` +passes them to the tests. To run a suite against your own deployment, set them +yourself. A fixture fails the test when a variable it needs is missing; it +never skips it. Add a row when you add a variable. + +| Variable | Set by | Read by | Meaning | +|---|---|---|---| +| `OPENSHELL_TEST_CONTAINER_ENGINE` | Every Docker and Podman base environment (`playbooks/docker.yaml`, `podman-rootful.yaml`, `podman-rootless.yaml`); `e2e:oci-image` tasks | `container-engine-support` | The command that reaches the gateway's container engine, split on whitespace, such as `docker` or `sudo -n podman`. | +| `OPENSHELL_TEST_PODMAN_USERNS` | `-userns-` (`playbooks/drivers/podman/userns.yaml`) | `podman-support` | The gateway's Podman user-namespace mode: `default`, `auto`, `keep-id` or `private`. | +| `OPENSHELL_TEST_CDI_SCENARIOS` | `-cdi` (`playbooks/cdi.yaml`) | `tests/suites/features/cdi` | The path of the document that names the CDI device kind, each scenario and the checks it expects. | +| `OPENSHELL_TEST_KEYCLOAK_ISSUER` | `-keycloak` (`playbooks/keycloak.yaml`) | `keycloak-support` | The OIDC issuer URL of the test realm, reachable from the test process. | +| `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | +| `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | +| `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | +| `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | +| `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | +| `OPENSHELL_TEST_POSTGRES_URL` | `tasks/scripts/run-postgres-tests.sh` (`mise run test:rust:postgres`), or you | `openshell-server`'s PostgreSQL tests | A disposable PostgreSQL database. Never point it at one a gateway uses. | + +Other `OPENSHELL_TEST_*` names are internal test hooks and are left out on +purpose: unit-test child-process markers, the re-entry markers in +`tests/support/environment.rs`, Nix guest-cache settings, `install.sh` test +hooks and crate test settings. + +### Addresses the environment chooses + +When a test must listen on an address the environment picked, because the +gateway or another service is configured to call it, the environment +publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it. The +test never picks the address itself. Tests that bind the same address can't +run at the same time. From 174cf4953f35f0602b4ee885b0c4609091899315 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 38/63] test(tmachine): add a gateway_overrides role for environment gateway settings Signed-off-by: Matthew Grossman --- TESTING.md | 29 ++++++++--- .../playbooks/drivers/podman/userns.yaml | 42 ++++----------- .../roles/cdi_fixtures/tasks/main.yaml | 24 ++------- .../gateway_overrides/defaults/main.yaml | 5 ++ .../roles/gateway_overrides/tasks/main.yaml | 51 +++++++++++++++++++ .../openshell_external_driver/tasks/main.yaml | 17 ++----- .../files/merge_gateway_toml_test.py | 22 ++++++++ .../tasks/merge-settings.yaml | 7 +-- 8 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml diff --git a/TESTING.md b/TESTING.md index e234086747..7829434e28 100644 --- a/TESTING.md +++ b/TESTING.md @@ -324,7 +324,9 @@ service. clients there from the variables. A missing variable fails the test; it never skips it. - **Documentation.** Document the variables in the fixture's module docs, so - anyone can run the suite against their own deployment by setting them. + anyone can run the suite against their own deployment by setting them, and + add a row for each to the + [environment contract table](tests/support/README.md#environment-contract). The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it with: @@ -596,11 +598,26 @@ nix run .#openshell-test-runner -- run \ #### Gateway overrides in an environment -An environment overrides gateway settings by writing the tables to change to -`/etc/openshell-test/gateway-overrides.toml` in a playbook that runs before the -installer. The file holds whole tables, such as `[openshell.gateway]` or -`[openshell.drivers.podman]`, and is not a complete configuration. Each -installer applies it to the gateway's own file: +An environment overrides gateway settings with the `gateway_overrides` role, +which takes a name and the tables to change, such as `[openshell.gateway]` or +`[openshell.drivers.podman]`: + +```yaml +- name: Push configuration to supervisors + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: config-push + gateway_overrides_content: | + [openshell.gateway] + config_delivery_mode = "push" +``` + +The role writes each name as its own marked block in +`/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's +settings. Two blocks must not declare the same table. Don't write the file +directly. The file is not a complete configuration. Each installer applies it +to the gateway's own file: - `binaries` applies it to the `/etc/openshell/gateway.toml` it generates. - `deb` and `rpm` apply it to the service user's diff --git a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml index 202270100f..dc74fe8a93 100644 --- a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml +++ b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml @@ -4,18 +4,16 @@ # Configure the Podman gateway's user-namespace mode and publish it. # # `podman_userns_mode` is default, auto, keep-id or private. Every mode except -# default writes its `[openshell.drivers.podman]` settings to -# `/etc/openshell-test/gateway-overrides.toml`, which every installer applies to the active -# gateway-overrides.toml (the openshell_gateway role for `binaries`, the -# openshell_packaged_gateway role for the packages); default removes the file. The -# Podman support crate (tests/support/podman) reads the published mode and -# runs Podman's own reference container in it. +# default adds its `[openshell.drivers.podman]` settings to the gateway +# overrides (the gateway_overrides role), which every installer applies to the +# gateway's configuration. Default mode adds nothing, so userns, uidmap and +# gidmap stay unset. The Podman support crate (tests/support/podman) reads the +# published mode and runs Podman's own reference container in it. --- - name: Podman user namespaces for tests hosts: all gather_facts: false vars: - podman_userns_driver_config: /etc/openshell-test/gateway-overrides.toml podman_userns_settings: default: "" auto: | @@ -43,34 +41,14 @@ {{ podman_userns_settings.keys() | join(', ') }}, not {{ podman_userns_mode | default('unset') }} - - name: Create the test configuration directory - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config | dirname }}" - state: directory - owner: root - group: root - mode: "0755" - - name: Write the Podman driver user-namespace settings - become: true - ansible.builtin.copy: - dest: "{{ podman_userns_driver_config }}" - content: "{{ podman_userns_settings[podman_userns_mode] }}" - owner: root - group: root - mode: "0644" + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: podman-userns + gateway_overrides_content: "{{ podman_userns_settings[podman_userns_mode] }}" when: podman_userns_mode != "default" - # The fragment is the only source of userns, uidmap and gidmap settings in - # the gateway's configuration, so default mode leaves them unset. - - name: Remove Podman driver user-namespace settings for default mode - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config }}" - state: absent - when: podman_userns_mode == "default" - - name: Publish Podman user-namespace test variables ansible.builtin.include_tasks: ../../test-env.yaml vars: diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml index 5f0c04586c..250629d90b 100644 --- a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml @@ -38,25 +38,11 @@ # The tests select the scenario device per sandbox through driver config, which # the gateway rejects unless an administrator allows it. The installers add # this to the active driver's settings. -- name: Create the test settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - owner: root - group: root - mode: "0755" - -# The gateway must accept a per-sandbox CDI device, which it does only when the -# driver allows caller driver config. Every installer merges this override into -# the gateway's own configuration (see openshell_gateway/tasks/merge-settings.yaml). - name: Allow caller driver config for the CDI tests - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: cdi + gateway_overrides_content: | [openshell.drivers.{{ environment_driver }}] allow_driver_config = true - owner: root - group: root - mode: "0644" diff --git a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml new file mode 100644 index 0000000000..73e224fb07 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +gateway_overrides_path: /etc/openshell-test/gateway-overrides.toml diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml new file mode 100644 index 0000000000..5114e90e69 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Add gateway settings to /etc/openshell-test/gateway-overrides.toml, which +# the `binaries`, `deb` and `rpm` installers apply to the gateway's +# configuration (openshell_gateway/tasks/merge-settings.yaml; k3s installers +# ignore it). +# +# Inputs: +# gateway_overrides_name names the marked block, such as `cdi`; writing +# the same name again replaces its block +# gateway_overrides_content whole TOML tables, such as +# `[openshell.drivers.podman]` +# +# Each name gets its own marked block, so writers keep each other's settings +# and may run in any order. Two blocks must not declare the same table: TOML +# rejects a table defined twice, and the installer fails on it. +# +# TODO: https://github.com/NVIDIA/OpenShell/pull/5489 reads gateway.d drop-in +# files. When it lands, write each name to its own drop-in here instead. +--- +- name: Require a gateway override name and content + ansible.builtin.assert: + that: + - gateway_overrides_name is defined + - gateway_overrides_name is match('^[a-z0-9][a-z0-9-]*$') + - gateway_overrides_content is defined + - gateway_overrides_content | trim | length > 0 + fail_msg: >- + gateway_overrides needs gateway_overrides_name (lowercase letters, + digits and dashes) and non-empty gateway_overrides_content + +- name: Create the test configuration directory + become: true + ansible.builtin.file: + path: "{{ gateway_overrides_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Write the {{ gateway_overrides_name }} gateway overrides + become: true + ansible.builtin.blockinfile: + path: "{{ gateway_overrides_path }}" + create: true + owner: root + group: root + mode: "0644" + marker: "# {mark} {{ gateway_overrides_name }}" + block: "{{ gateway_overrides_content }}" diff --git a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml index a31363d57d..2f3d4e3872 100644 --- a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml @@ -108,18 +108,11 @@ ansible.builtin.fail: msg: The external driver socket did not appear -- name: Create the environment settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - mode: "0755" - - name: Point the gateway at the external driver - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - mode: "0644" - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: external-driver + gateway_overrides_content: | [openshell.drivers.{{ openshell_external_driver_name }}] socket_path = "{{ openshell_external_driver_socket }}" diff --git a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py index ba6926169a..3f37c15178 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py +++ b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py @@ -98,3 +98,25 @@ def test_creates_from_default_else_minimal(tmp_path): run, base = merge(other, None, '[openshell.drivers.podman]\nuserns = "auto"\n') parsed = tomllib.loads(base.read_text()) assert parsed["openshell"]["version"] == 2 + + +def test_applies_every_marked_block(tmp_path): + overrides = ( + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + "# BEGIN config-push\n[openshell.gateway]\nconfig_delivery_mode = \"push\"\n# END config-push\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode == 0, run.stderr + parsed = tomllib.loads(base.read_text()) + assert parsed["openshell"]["drivers"]["podman"]["userns"] == "auto" + assert parsed["openshell"]["gateway"]["config_delivery_mode"] == "push" + + +def test_rejects_blocks_that_declare_the_same_table(tmp_path): + overrides = ( + "# BEGIN cdi\n[openshell.drivers.podman]\nallow_driver_config = true\n# END cdi\n" + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode != 0 + assert base.read_text() == BASE diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml index 6d167a3ec8..2b3b8e3e22 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml @@ -1,10 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# An environment overrides gateway settings by writing the tables to change, +# An environment overrides gateway settings by adding the tables to change, # such as `[openshell.drivers.podman]`, to -# /etc/openshell-test/gateway-overrides.toml before the installer runs (for -# example playbooks/drivers/podman/userns.yaml). Apply them to +# /etc/openshell-test/gateway-overrides.toml with the gateway_overrides role +# before the installer runs (for example playbooks/drivers/podman/userns.yaml). +# Apply them to # `openshell_gateway_merge_target` without replacing the file; see # files/merge-gateway-toml.py. # From 2eb7ee9124fce3adbdaba5359b332964354c687d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:00:13 -0800 Subject: [PATCH 39/63] test(testkit): serve TLS from host servers with a test or environment CA Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 4 + tests/suites/features/Cargo.lock | 4 + tests/support/Cargo.lock | 4 + tests/support/README.md | 2 + tests/support/host-server/Cargo.toml | 4 + tests/support/host-server/src/http.rs | 72 ++++- tests/support/host-server/src/lib.rs | 5 +- tests/support/host-server/src/tls.rs | 308 +++++++++++++++++++++ tests/support/host-server/src/websocket.rs | 3 +- 9 files changed, 391 insertions(+), 15 deletions(-) create mode 100644 tests/support/host-server/src/tls.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 0d17b24979..53c4bdee9e 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1285,9 +1285,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 3a0b5a30b2..2f966d222c 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1317,9 +1317,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 044a44aa3e..440b9ec84b 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1281,9 +1281,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/README.md b/tests/support/README.md index f2ca63d396..c13ba758e1 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -23,6 +23,8 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | +| `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | | `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | | `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index cf51fd6a4c..ccac94f0d1 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -12,9 +12,13 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } +rcgen = { version = "0.14.10", default-features = false, features = ["pem"] } +rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } serde_json = "1" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["tls12"] } tungstenite = "0.26" [dev-dependencies] diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index b93e6dddcc..8b1f830faa 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -7,10 +7,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::TcpStream; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; use tokio::task::JoinHandle; +use tokio_rustls::TlsAcceptor; +use crate::tls::Identity; use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; /// A request an [`HttpRecorder`] read. A chunked body arrives decoded. @@ -171,6 +172,30 @@ impl HttpRecorder { /// Answer every request with `handler`'s response. pub async fn serve( handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(None, handler).await + } + + /// Answer every request over TLS with `200` and `body`, presenting + /// `identity`'s certificate. + pub async fn start_tls(identity: &Identity, body: impl Into) -> Result { + let body = body.into(); + Self::serve_tls(identity, move |_| Response::text(body.clone())).await + } + + /// Answer every request over TLS with `handler`'s response, presenting + /// `identity`'s certificate. A connection that fails the TLS handshake + /// counts in [`connections`](Self::connections) but records no request. + pub async fn serve_tls( + identity: &Identity, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(Some(identity.acceptor()?), handler).await + } + + async fn listen( + tls: Option, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { let (listener, port) = bind().await?; let handler: Arc = Arc::new(handler); @@ -181,11 +206,21 @@ impl HttpRecorder { let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { task_connections.fetch_add(1, Ordering::AcqRel); - tokio::spawn(respond( - stream, - Arc::clone(&handler), - Arc::clone(&task_requests), - )); + let _ = stream.set_nodelay(true); + let local_addr = stream.local_addr().ok(); + let handler = Arc::clone(&handler); + let requests = Arc::clone(&task_requests); + let tls = tls.clone(); + tokio::spawn(async move { + match tls { + None => respond(stream, local_addr, handler, requests).await, + Some(tls) => { + if let Ok(stream) = tls.accept(stream).await { + respond(stream, local_addr, handler, requests).await; + } + } + } + }); } }); Ok(Self { @@ -239,10 +274,14 @@ impl Drop for HttpRecorder { /// Read requests, record each, and answer with `handler` until a response /// or request closes the connection. -async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc>>) { - let _ = stream.set_nodelay(true); +async fn respond( + mut stream: impl AsyncRead + AsyncWrite + Unpin, + local_addr: Option, + handler: Arc, + requests: Arc>>, +) { let mut buffer = Vec::new(); - while let Some(request) = read_request(&mut stream, &mut buffer).await { + while let Some(request) = read_request(&mut stream, local_addr, &mut buffer).await { let response = handler(&request); let close = !response.keep_alive || request @@ -260,13 +299,18 @@ async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc) -> Option { +/// `buffer`, and leave any bytes after it in `buffer`. `local_addr` is the +/// address the connection arrived on. +pub(crate) async fn read_request( + stream: &mut (impl AsyncRead + Unpin), + local_addr: Option, + buffer: &mut Vec, +) -> Option { let mut chunk = [0_u8; 4096]; loop { if let Some((mut request, length)) = parse_request(buffer) { buffer.drain(..length); - request.local_addr = stream.local_addr().ok(); + request.local_addr = local_addr; return Some(request); } match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { @@ -337,6 +381,8 @@ fn decode_chunked(bytes: &[u8]) -> Option<(Vec, usize)> { #[cfg(test)] mod tests { + use tokio::net::TcpStream; + use super::*; async fn connect(port: u16) -> TcpStream { diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index e9781bf502..374d3be419 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -11,7 +11,8 @@ //! host, such as one on a remote cluster, can't run tests that use them. //! //! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's -//! [`Response`], and records each parsed [`Request`]. +//! [`Response`], and records each parsed [`Request`]. It serves HTTPS with +//! a certificate from a [`TestCa`]. //! - [`WebSocketRecorder`] completes the WebSocket handshake and records the //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the @@ -21,6 +22,7 @@ mod http; mod tcp; +mod tls; mod websocket; use std::time::Duration; @@ -30,6 +32,7 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request, Response}; pub use tcp::{TcpServer, exchange}; +pub use tls::{ADDITIONAL_CA_CERT_ENV, ADDITIONAL_CA_KEY_ENV, Identity, TestCa}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; /// The name a sandbox reaches these servers by. diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs new file mode 100644 index 0000000000..6b36293b92 --- /dev/null +++ b/tests/support/host-server/src/tls.rs @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Certificate authorities for TLS host servers. +//! +//! A test signs a server certificate with a [`TestCa`] and serves it with +//! [`HttpRecorder::serve_tls`](crate::HttpRecorder::serve_tls): +//! +//! - [`TestCa::new`] makes a CA that only the test knows, so nothing trusts it +//! unless the test hands over [`TestCa::cert_pem`]. +//! - [`TestCa::from_env`] loads the environment's additional CA, which the +//! gateway is configured to trust (`additional_ca_bundle`). The environment +//! publishes the paths of its PEM certificate and key in +//! `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY`. + +use std::path::Path; +use std::sync::Arc; + +use openshell_crypto::pki::{self, KeyPair}; +use rcgen::{ + BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, + KeyUsagePurpose, +}; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use testkit::{Error, Result}; +use tokio_rustls::TlsAcceptor; + +/// The path of the environment's additional CA certificate, in PEM. +pub const ADDITIONAL_CA_CERT_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_CERT"; +/// The path of the environment's additional CA private key, in PEM. +pub const ADDITIONAL_CA_KEY_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_KEY"; + +/// A certificate authority that signs server certificates for host servers. +pub struct TestCa { + cert_pem: String, + issuer: Issuer<'static, KeyPair>, +} + +impl TestCa { + /// A new CA that only this test knows. + pub fn new() -> Result { + let key = pki::generate_keypair().map_err(failed("generate a test CA key"))?; + let cert = pki::self_signed(ca_params(), &key).map_err(failed("sign the test CA"))?; + Ok(Self { + cert_pem: cert.pem(), + issuer: cert.into_issuer(key), + }) + } + + /// The environment's additional CA, from the files that + /// `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY` + /// name. Fails if either is unset. + pub fn from_env() -> Result { + Self::from_vars(|name| std::env::var(name).ok()) + } + + fn from_vars(var: impl Fn(&str) -> Option) -> Result { + let path = |name: &str| { + var(name) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + Error::msg(format!( + "{name} is not set; run this test in an environment with an additional CA" + )) + }) + }; + let cert = path(ADDITIONAL_CA_CERT_ENV)?; + let key = path(ADDITIONAL_CA_KEY_ENV)?; + Self::from_files(cert.as_ref(), key.as_ref()) + } + + /// A CA from its PEM certificate and private key files. + pub fn from_files(cert: &Path, key: &Path) -> Result { + let read = |path: &Path| { + std::fs::read_to_string(path) + .map_err(|error| Error::msg(format!("read {}: {error}", path.display()))) + }; + let cert_pem = read(cert)?; + let key_pem = read(key)?; + let der = CertificateDer::from_pem_slice(cert_pem.as_bytes()).map_err(failed(format!( + "parse the CA certificate {}", + cert.display() + )))?; + let key = KeyPair::from_pem(&key_pem) + .map_err(failed(format!("parse the CA key {}", key.display())))?; + let issuer = pki::issuer_from_der(&der, key) + .map_err(failed(format!("load the CA {}", cert.display())))?; + Ok(Self { cert_pem, issuer }) + } + + /// The CA certificate, in PEM, for a client that should trust it. + pub fn cert_pem(&self) -> &str { + &self.cert_pem + } + + /// A server certificate for `names`, signed by this CA. A name that parses + /// as an IP address becomes an IP address entry. + pub fn leaf(&self, names: &[&str]) -> Result { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let first = names + .first() + .cloned() + .ok_or_else(|| Error::msg("a server certificate needs at least one name"))?; + let mut params = CertificateParams::new(names.clone()) + .map_err(failed(format!("server certificate names {names:?}")))?; + params.distinguished_name.push(DnType::CommonName, first); + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let key = pki::generate_keypair().map_err(failed("generate a server key"))?; + let cert = pki::signed_by_issuer(params, &key, &self.issuer) + .map_err(failed(format!("sign a server certificate for {names:?}")))?; + let key_der = key + .serialize_der() + .map_err(failed("export the server key"))?; + let key_pem = key + .serialize_pem() + .map_err(failed("export the server key"))?; + Ok(Identity { + cert_pem: cert.pem(), + key_pem, + cert_der: cert.der().clone(), + key_der, + }) + } +} + +/// A server certificate and its private key. +#[derive(Clone)] +pub struct Identity { + pub cert_pem: String, + pub key_pem: String, + cert_der: CertificateDer<'static>, + key_der: Vec, +} + +impl Identity { + pub(crate) fn acceptor(&self) -> Result { + let key = PrivateKeyDer::Pkcs8(self.key_der.clone().into()); + let mut config = openshell_crypto::tls::try_server_builder() + .map_err(failed("build a TLS server configuration"))? + .with_no_client_auth() + .with_single_cert(vec![self.cert_der.clone()], key) + .map_err(failed("load the server certificate"))?; + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(config))) + } +} + +impl std::fmt::Debug for Identity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Identity") + .field("cert_pem", &self.cert_pem) + .finish_non_exhaustive() + } +} + +fn ca_params() -> CertificateParams { + let mut params = CertificateParams::default(); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell test CA"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + KeyUsagePurpose::DigitalSignature, + ]; + params +} + +fn failed(context: impl std::fmt::Display) -> impl FnOnce(E) -> Error { + move |error| Error::msg(format!("{context}: {error}")) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rustls::pki_types::ServerName; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + use tokio_rustls::client::TlsStream; + + use super::*; + use crate::{HttpRecorder, Response}; + + /// Open a TLS connection to `port` on loopback as `name`, trusting `ca`. + async fn connect(ca: &str, name: &str, port: u16) -> std::io::Result> { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.as_bytes()).unwrap()) + .unwrap(); + let config = openshell_crypto::tls::try_client_builder() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await?; + stream.set_nodelay(true)?; + TlsConnector::from(Arc::new(config)) + .connect(ServerName::try_from(name.to_string()).unwrap(), stream) + .await + } + + async fn get(stream: &mut TlsStream, path: &str) -> String { + stream + .write_all(format!("GET {path} HTTP/1.1\r\nHost: test\r\n\r\n").as_bytes()) + .await + .unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + String::from_utf8_lossy(&response).into_owned() + } + + fn write_ca(dir: &str) -> (PathBuf, PathBuf, String) { + let key = pki::generate_keypair().unwrap(); + let cert = pki::self_signed(ca_params(), &key).unwrap(); + let dir = std::env::temp_dir().join(format!("{dir}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (cert_path, key_path) = (dir.join("ca.pem"), dir.join("ca-key.pem")); + std::fs::write(&cert_path, cert.pem()).unwrap(); + std::fs::write(&key_path, key.serialize_pem().unwrap()).unwrap(); + (cert_path, key_path, cert.pem()) + } + + #[tokio::test] + async fn serve_tls_answers_a_client_that_trusts_the_ca() { + let ca = TestCa::new().unwrap(); + let identity = ca.leaf(&["localhost", "127.0.0.1"]).unwrap(); + let recorder = HttpRecorder::serve_tls(&identity, |request| { + Response::text(format!("tls {}", request.path())) + }) + .await + .unwrap(); + for name in ["localhost", "127.0.0.1"] { + let mut stream = connect(ca.cert_pem(), name, recorder.port).await.unwrap(); + let response = get(&mut stream, "/hello").await; + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.ends_with("tls /hello"), "{response}"); + } + let requests = recorder.wait_for_requests(2).await.unwrap(); + assert_eq!(requests[0].path(), "/hello"); + } + + #[tokio::test] + async fn serve_tls_is_rejected_for_another_name_or_ca() { + let ca = TestCa::new().unwrap(); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "ok") + .await + .unwrap(); + let wrong_name = connect(ca.cert_pem(), "other.test", recorder.port).await; + assert!( + wrong_name.is_err(), + "a certificate for another name was accepted" + ); + let other = TestCa::new().unwrap(); + let wrong_ca = connect(other.cert_pem(), "localhost", recorder.port).await; + assert!( + wrong_ca.is_err(), + "a certificate from another CA was accepted" + ); + assert!(recorder.requests().is_empty()); + } + + #[tokio::test] + async fn from_vars_loads_the_environment_ca() { + let (cert, key, cert_pem) = write_ca("host-server-env-ca"); + let ca = TestCa::from_vars(|name| match name { + ADDITIONAL_CA_CERT_ENV => Some(cert.display().to_string()), + ADDITIONAL_CA_KEY_ENV => Some(key.display().to_string()), + _ => None, + }) + .unwrap(); + assert_eq!(ca.cert_pem(), cert_pem); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "env") + .await + .unwrap(); + let mut stream = connect(&cert_pem, "localhost", recorder.port) + .await + .unwrap(); + assert!(get(&mut stream, "/").await.ends_with("env")); + } + + #[test] + fn from_vars_fails_when_a_variable_is_missing() { + let error = TestCa::from_vars(|name| { + (name == ADDITIONAL_CA_CERT_ENV).then(|| "/nonexistent/ca.pem".to_string()) + }) + .err() + .unwrap(); + assert!(error.to_string().contains(ADDITIONAL_CA_KEY_ENV), "{error}"); + let error = TestCa::from_vars(|_| None).err().unwrap(); + assert!( + error.to_string().contains(ADDITIONAL_CA_CERT_ENV), + "{error}" + ); + } + + #[test] + fn leaf_needs_a_name() { + assert!(TestCa::new().unwrap().leaf(&[]).is_err()); + } +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 4f38bdb8f3..11ac2606f4 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -105,7 +105,8 @@ impl Drop for WebSocketRecorder { async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream, &mut Vec::new()).await else { + let local_addr = stream.local_addr().ok(); + let Some(request) = read_request(&mut stream, local_addr, &mut Vec::new()).await else { return; }; let Some(key) = header(&request.head, "sec-websocket-key") else { From bd9f62531830406c8a7522fbd7739f635971adae Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:44:31 -0800 Subject: [PATCH 40/63] feat(testkit): add workspaces, token clients, and per-CLI environment Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 6 + tests/testkit/testkit/src/cli.rs | 15 ++- tests/testkit/testkit/src/context.rs | 5 + tests/testkit/testkit/src/gateway.rs | 26 ++++- tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/workspace.rs | 147 +++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 tests/testkit/testkit/src/workspace.rs diff --git a/tests/testkit/README.md b/tests/testkit/README.md index a0c5642593..a2f4fe63e8 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -117,6 +117,12 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `Cli::apply_env` to point another program at the same gateway, such as `ssh` with the `ProxyCommand` from `sandbox ssh-config`. `Cli::gateway` says which gateway that is. +- `Cli::with_env(name, value)` for a CLI with extra environment on every + run, such as `XDG_CONFIG_HOME` holding another user's session. +- `cx.cli_handle().gateway().connect_with_token(token)` for an SDK client that sends a + bearer token, such as an OIDC access token, and no client certificate. +- `cx.workspace(suffix)` to create a workspace with cleanup. Its + `add_member(subject, role)` and `remove_member(subject)` manage members. - `cx.on_cleanup(name, hook)` to remove something when the test ends. - `testkit::policy` for sandbox policies: `policy::base()` reads `testkit/src/base-policy.yaml`, which allows no network access, and diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index 2973ea045b..ecc49dacc2 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -8,7 +8,7 @@ //! those and observe through the SDK where they can. Rely on the CLI's exit //! code and structured output (`--output json`), not on its table text. -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -30,6 +30,7 @@ pub(crate) const BIN_ENV: &str = "OPENSHELL_BIN"; pub struct Cli { binary: PathBuf, gateway: Gateway, + env: Vec<(OsString, OsString)>, } /// The buffered result of one CLI invocation. @@ -50,9 +51,19 @@ impl Cli { Self { binary, gateway: gateway.clone(), + env: Vec::new(), } } + /// This CLI with `name` set to `value` in its environment on every run, + /// such as `XDG_CONFIG_HOME` for a CLI that reads another user's + /// credentials. The gateway and workspace selection still wins. + #[must_use] + pub fn with_env(mut self, name: impl Into, value: impl Into) -> Self { + self.env.push((name.into(), value.into())); + self + } + /// `binary` in place of the CLI, pointed at an unregistered gateway. #[cfg(test)] pub(crate) fn with_binary(binary: &str) -> Self { @@ -62,6 +73,7 @@ impl Cli { "http://127.0.0.1:1".to_string(), "default".to_string(), ), + env: Vec::new(), } } @@ -172,6 +184,7 @@ impl Cli { /// `ssh` with the `ProxyCommand` from `sandbox ssh-config`. pub fn apply_env(&self, process: &mut tokio::process::Command) { process + .envs(self.env.iter().map(|(name, value)| (name, value))) .env("NO_COLOR", "1") .env(WORKSPACE_ENV, &self.gateway.workspace); // Select exactly the gateway the SDK connected to. diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index cb80cd65be..b12c55a420 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -171,6 +171,11 @@ impl Context { &self.handles.client } + /// The SDK client for the whole gateway, not scoped to a workspace. + pub(crate) fn gateway_client(&self) -> &OpenShellClient { + &self.handles.grpc.client + } + /// A resource name unique to this run: `test--`. /// /// Sandbox names can't exceed [`MAX_SANDBOX_NAME_LEN`], so a sandbox diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index 79bd5fe827..5773f924c8 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -17,7 +17,9 @@ use std::path::{Path, PathBuf}; use std::time::Duration; -use openshell_sdk::{ClientConfig, ClientIdentity, Health, OpenShellClient, ServiceStatus}; +use openshell_sdk::{ + AuthConfig, ClientConfig, ClientIdentity, Health, OpenShellClient, ServiceStatus, +}; use crate::error::{Error, Result, ResultExt}; @@ -121,9 +123,10 @@ impl Gateway { }) } - /// The directory holding the gateway's CA and client certificate, for - /// an mTLS gateway. - pub(crate) fn mtls_dir(&self) -> Option<&Path> { + /// The directory holding the gateway's CA (`ca.crt`) and the client + /// certificate and key (`tls.crt`, `tls.key`) for mTLS, or `None` for a + /// gateway testkit reaches without a client certificate. + pub fn mtls_dir(&self) -> Option<&Path> { self.mtls_dir.as_deref() } @@ -163,6 +166,21 @@ impl Gateway { .context(format!("connect to {}", self.endpoint)) } + /// Connect to the gateway as the user `token` names, a bearer token such + /// as an OIDC access token, or with no credentials for `None`. The + /// connection verifies the gateway with its CA and presents no client + /// certificate, so the gateway sees only the token. + pub async fn connect_with_token(&self, token: Option<&str>) -> Result { + let mut config = ClientConfig::new(self.endpoint.clone()); + if let Some(dir) = &self.mtls_dir { + config.ca_cert = Some(read(&dir.join("ca.crt"))?); + } + config.auth = token.map(AuthConfig::oidc); + OpenShellClient::connect(config) + .await + .context(format!("connect to {} with a token", self.endpoint)) + } + /// Wait until the gateway accepts a connection and reports itself /// healthy, retrying until `timeout`. /// diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 65b7aa799e..7a67a42ac3 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -45,6 +45,7 @@ mod ready; mod registry; mod run; pub mod service; +mod workspace; mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; @@ -70,6 +71,7 @@ pub use run::{TestInfo, run}; pub use service::{SERVICE_REQUEST_TIMEOUT, ServiceResponse}; pub use testkit_macros::test; pub use tonic::Code as GrpcCode; +pub use workspace::Workspace; pub use yaml_file::YamlFile; #[doc(hidden)] diff --git a/tests/testkit/testkit/src/workspace.rs b/tests/testkit/testkit/src/workspace.rs new file mode 100644 index 0000000000..37263a6bf4 --- /dev/null +++ b/tests/testkit/testkit/src/workspace.rs @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workspaces a test creates, and their members. + +use std::collections::HashMap; +use std::time::{Duration, Instant}; + +use openshell_sdk::raw::proto; +use openshell_sdk::{DeleteOptions, OpenShellClient}; + +use crate::context::Context; +use crate::error::{Error, Result, ResultExt as _}; + +/// The longest workspace name the gateway accepts. +const MAX_WORKSPACE_NAME_LEN: usize = 19; + +/// How long cleanup waits for a workspace's resources to go before giving +/// up on deleting it. Under [`crate::CLEANUP_TIMEOUT`], so the failure +/// names the workspace. +const DELETE_RETRY_TIMEOUT: Duration = Duration::from_secs(90); +const DELETE_RETRY_INTERVAL: Duration = Duration::from_secs(2); + +impl Context { + /// Create a workspace named [`Context::name`]`(suffix)`, as the gateway + /// client testkit connects with. The suffix is at most five characters, + /// to fit the gateway's limit on workspace names. Cleanup deletes the + /// workspace after the test's sandboxes, waiting for resources still in + /// it to go. + pub async fn workspace(&self, suffix: &str) -> Result { + let name = self.name(suffix); + crate::ensure!( + name.len() <= MAX_WORKSPACE_NAME_LEN, + "workspace name {name} exceeds {MAX_WORKSPACE_NAME_LEN} characters; use a shorter suffix than {suffix:?}" + ); + let client = self.gateway_client().clone(); + // Deleting a missing workspace succeeds, so register the delete + // before the create. + let cleanup_client = client.clone(); + let owned = name.clone(); + self.on_cleanup(format!("delete workspace {name}"), move || { + delete_workspace(cleanup_client, owned) + }); + client + .create_workspace(&name, HashMap::new()) + .await + .context(format!("create workspace {name}"))?; + Ok(Workspace { name, client }) + } +} + +/// A workspace from [`Context::workspace`]. +#[derive(Clone)] +pub struct Workspace { + name: String, + client: OpenShellClient, +} + +impl Workspace { + pub fn name(&self) -> &str { + &self.name + } + + /// Make `subject`, an OIDC subject claim, a member with `role`. + pub async fn add_member(&self, subject: &str, role: proto::WorkspaceRole) -> Result { + self.client + .raw_grpc_fresh() + .await? + .add_workspace_member(proto::AddWorkspaceMemberRequest { + workspace_scope: Some(proto::workspace_selector(&self.name)), + principal_subject: subject.to_string(), + role: role.into(), + request_id: String::new(), + }) + .await + .context(format!( + "add {subject} to workspace {} as {}", + self.name, + role.as_str_name() + ))?; + Ok(()) + } + + /// Remove `subject` from the workspace's members. + pub async fn remove_member(&self, subject: &str) -> Result { + self.client + .raw_grpc_fresh() + .await? + .remove_workspace_member(proto::RemoveWorkspaceMemberRequest { + workspace_scope: Some(proto::workspace_selector(&self.name)), + principal_subject: subject.to_string(), + allow_missing: false, + request_id: String::new(), + }) + .await + .context(format!("remove {subject} from workspace {}", self.name))?; + Ok(()) + } +} + +/// Delete `name`, retrying while resources the gateway is still deleting +/// block it. +async fn delete_workspace(client: OpenShellClient, name: String) -> Result { + let started = Instant::now(); + loop { + let error = match client + .delete_workspace( + &name, + DeleteOptions { + allow_missing: true, + }, + ) + .await + { + Ok(_) => return Ok(()), + Err(error) => Error::from(error), + }; + if !still_has_resources(&error) || started.elapsed() >= DELETE_RETRY_TIMEOUT { + return Err(error).context(format!("delete workspace {name}")); + } + tokio::time::sleep(DELETE_RETRY_INTERVAL).await; + } +} + +fn still_has_resources(error: &Error) -> bool { + error.grpc_code() == Some(tonic::Code::FailedPrecondition) + && error.to_string().contains("still contains resources") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn retries_only_a_workspace_with_resources() { + let busy = Error::from(tonic::Status::failed_precondition( + "workspace 'w' still contains resources: sandbox", + )); + let other = Error::from(tonic::Status::failed_precondition( + "workspace is terminating", + )); + let denied = Error::from(tonic::Status::permission_denied("still contains resources")); + assert!(still_has_resources(&busy)); + assert!(!still_has_resources(&other)); + assert!(!still_has_resources(&denied)); + } +} From e80b461173fbae2b00a9b15e7fe9fc40afeedda9 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:50:26 -0800 Subject: [PATCH 41/63] feat(testkit): let a CLI change its environment and workspace Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 6 ++++-- tests/testkit/testkit/src/cli.rs | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index a2f4fe63e8..f98cf31245 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -117,8 +117,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `Cli::apply_env` to point another program at the same gateway, such as `ssh` with the `ProxyCommand` from `sandbox ssh-config`. `Cli::gateway` says which gateway that is. -- `Cli::with_env(name, value)` for a CLI with extra environment on every - run, such as `XDG_CONFIG_HOME` holding another user's session. +- `Cli::with_env(name, value)` and `Cli::without_env(name)` for a CLI with + changed environment on every run, such as `XDG_CONFIG_HOME` holding + another user's session. `Cli::in_workspace(name)` points it at another + workspace. - `cx.cli_handle().gateway().connect_with_token(token)` for an SDK client that sends a bearer token, such as an OIDC access token, and no client certificate. - `cx.workspace(suffix)` to create a workspace with cleanup. Its diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index ecc49dacc2..831136dbe1 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -30,7 +30,8 @@ pub(crate) const BIN_ENV: &str = "OPENSHELL_BIN"; pub struct Cli { binary: PathBuf, gateway: Gateway, - env: Vec<(OsString, OsString)>, + /// Variables to set, or with `None`, to remove, on every run. + env: Vec<(OsString, Option)>, } /// The buffered result of one CLI invocation. @@ -60,7 +61,23 @@ impl Cli { /// credentials. The gateway and workspace selection still wins. #[must_use] pub fn with_env(mut self, name: impl Into, value: impl Into) -> Self { - self.env.push((name.into(), value.into())); + self.env.push((name.into(), Some(value.into()))); + self + } + + /// This CLI with `name` removed from its environment on every run, such + /// as a variable the test process inherits that changes how the CLI + /// logs in. + #[must_use] + pub fn without_env(mut self, name: impl Into) -> Self { + self.env.push((name.into(), None)); + self + } + + /// This CLI pointed at `workspace` on the same gateway. + #[must_use] + pub fn in_workspace(mut self, workspace: impl Into) -> Self { + self.gateway.workspace = workspace.into(); self } @@ -183,8 +200,13 @@ impl Cli { /// invocation is. Use it for a program that runs the CLI itself, such as /// `ssh` with the `ProxyCommand` from `sandbox ssh-config`. pub fn apply_env(&self, process: &mut tokio::process::Command) { + for (name, value) in &self.env { + match value { + Some(value) => process.env(name, value), + None => process.env_remove(name), + }; + } process - .envs(self.env.iter().map(|(name, value)| (name, value))) .env("NO_COLOR", "1") .env(WORKSPACE_ENV, &self.gateway.workspace); // Select exactly the gateway the SDK connected to. From 0b2e45030c57e09acc97cc25cdc5e362f0d8fca4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:50:26 -0800 Subject: [PATCH 42/63] feat(keycloak-support): add user sessions and the browser login Signed-off-by: Matthew Grossman --- tests/support/Cargo.lock | 3 + tests/support/keycloak/Cargo.toml | 5 +- tests/support/keycloak/src/browser.rs | 245 ++++++++++++++++++++++++++ tests/support/keycloak/src/lib.rs | 82 ++++++++- tests/support/keycloak/src/session.rs | 156 ++++++++++++++++ tests/support/keycloak/src/token.rs | 84 +++++++++ 6 files changed, 569 insertions(+), 6 deletions(-) create mode 100644 tests/support/keycloak/src/browser.rs create mode 100644 tests/support/keycloak/src/session.rs create mode 100644 tests/support/keycloak/src/token.rs diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 3aa41cd5e4..694ceca3be 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1800,9 +1800,12 @@ dependencies = [ name = "keycloak-support" version = "0.0.0" dependencies = [ + "base64 0.22.1", "openshell-testkit", "serde_json", + "tempfile", "tokio", + "url", ] [[package]] diff --git a/tests/support/keycloak/Cargo.toml b/tests/support/keycloak/Cargo.toml index c91cf2bc10..95de227f7b 100644 --- a/tests/support/keycloak/Cargo.toml +++ b/tests/support/keycloak/Cargo.toml @@ -13,5 +13,8 @@ publish = false [dependencies] openshell-testkit = { path = "../../testkit/testkit" } +base64 = "0.22" serde_json = "1" -tokio = { version = "1.43", features = ["io-util", "process"] } +tempfile = "3" +tokio = { version = "1.43", features = ["fs", "io-util", "process", "time"] } +url = "2" diff --git a/tests/support/keycloak/src/browser.rs b/tests/support/keycloak/src/browser.rs new file mode 100644 index 0000000000..ec48ec4afb --- /dev/null +++ b/tests/support/keycloak/src/browser.rs @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The CLI's browser (PKCE) login, with curl in place of the browser. +//! +//! The CLI opens the authorization URL with `xdg-open`. A stand-in earlier +//! on `PATH` records the URL instead, and curl signs in on Keycloak's login +//! form, following the redirect to the CLI's loopback callback. + +use std::os::unix::fs::PermissionsExt as _; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use tempfile::TempDir; +use testkit::{Cli, CliOutput, Context, Error, Poll, Result, ResultExt as _, poll_until}; +use tokio::process::Command; +use url::Url; + +use crate::{Keycloak, User}; + +const URL_FILE_ENV: &str = "OPENSHELL_TEST_BROWSER_URL_FILE"; +/// How long the CLI has to discover the issuer and open the browser. +const URL_TIMEOUT: Duration = Duration::from_secs(15); +/// How long the CLI has to exchange the code and save the session after the +/// callback. +const FINISH_TIMEOUT: Duration = Duration::from_secs(30); +const CURL_MAX_TIME: &str = "10"; + +/// A finished `openshell gateway add --oidc-issuer` browser login. +pub struct BrowserLogin { + /// The URL the CLI opened in the browser. + pub authorization_url: Url, + /// The CLI's output from `gateway add`. + pub output: CliOutput, + /// The page the CLI's callback served after the login. + pub callback_page: String, + cli: Cli, + gateway_name: String, + config_home: TempDir, +} + +/// Register the test's gateway under its own name in a fresh CLI +/// configuration with `gateway add --oidc-issuer`, and sign in as `user` in +/// the browser it opens, asking for `scopes`. Fails if the login doesn't +/// reach the CLI's callback; the test checks the rest. +pub async fn browser_login( + cx: &Context, + keycloak: &Keycloak, + user: User, + scopes: &str, +) -> Result { + let (username, password) = keycloak.credentials(user)?; + let base = cx.cli_handle(); + let gateway = base.gateway().clone(); + let gateway_name = gateway.name.clone().ok_or_else(|| { + Error::msg("the browser login registers a named gateway; testkit has only an endpoint") + })?; + + let config_home = tempfile::tempdir().context("create a CLI config directory")?; + let bin = config_home.path().join("bin"); + let url_file = config_home.path().join("browser-url"); + install_xdg_open_recorder(&bin)?; + let cli = base + .with_env("XDG_CONFIG_HOME", config_home.path()) + .with_env("HOME", config_home.path()) + .without_env("OPENSHELL_NO_BROWSER") + .without_env("OPENSHELL_OIDC_CLIENT_SECRET"); + let login = cli + .clone() + .with_env("PATH", prepend_path(&bin)?) + .with_env(URL_FILE_ENV, &url_file) + .spawn([ + "gateway", + "add", + &gateway.endpoint, + "--name", + &gateway_name, + "--local", + "--oidc-issuer", + keycloak.issuer(), + "--oidc-client-id", + keycloak.client_id(), + "--oidc-scopes", + scopes, + ])?; + + let authorization_url = wait_for_url(&url_file).await?; + let cookies = config_home.path().join("keycloak-cookies"); + let login_page = curl(&["--cookie-jar"], &cookies, authorization_url.as_str(), &[]) + .await + .context("load the Keycloak login page")?; + let action = login_form_action(&login_page)?; + let callback_page = curl( + &["--location", "--cookie-jar"], + &cookies, + &action, + &[ + &format!("username={username}"), + &format!("password={password}"), + "credentialId=", + ], + ) + .await + .context("sign in on the Keycloak login page")?; + let output = login.output(FINISH_TIMEOUT).await?; + testkit::ensure!( + callback_page.contains("Authentication successful"), + "the CLI's callback did not report success:\n{callback_page}\n{}{}", + output.stdout, + output.stderr + ); + Ok(BrowserLogin { + authorization_url, + output, + callback_page, + cli, + gateway_name, + config_home, + }) +} + +impl BrowserLogin { + /// The CLI's directory for the registered gateway, holding + /// `metadata.json` and `oidc_token.json`. + pub fn gateway_dir(&self) -> PathBuf { + self.config_home + .path() + .join("openshell") + .join("gateways") + .join(&self.gateway_name) + } + + /// The CLI signed in by this login. It trusts the gateway's CA, which an + /// OIDC registration doesn't save, so a test can make calls with it. + pub fn cli(&self, cx: &Context) -> Result { + if let Some(mtls) = cx.cli_handle().gateway().mtls_dir() { + let target = self.gateway_dir().join("mtls"); + std::fs::create_dir_all(&target).context(format!("create {}", target.display()))?; + std::fs::copy(mtls.join("ca.crt"), target.join("ca.crt")) + .context(format!("copy the gateway CA from {}", mtls.display()))?; + } + Ok(self.cli.clone()) + } +} + +fn install_xdg_open_recorder(bin: &Path) -> Result { + std::fs::create_dir_all(bin).context(format!("create {}", bin.display()))?; + let script = bin.join("xdg-open"); + std::fs::write( + &script, + format!("#!/bin/sh\nset -eu\nprintf '%s\\n' \"$1\" > \"${URL_FILE_ENV}\"\n"), + ) + .context("write the xdg-open stand-in")?; + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)) + .context("make the xdg-open stand-in executable") +} + +fn prepend_path(bin: &Path) -> Result { + let current = std::env::var_os("PATH").unwrap_or_default(); + std::env::join_paths(std::iter::once(bin.to_path_buf()).chain(std::env::split_paths(¤t))) + .map_err(|error| Error::msg(format!("build PATH: {error}"))) +} + +async fn wait_for_url(path: &Path) -> Result { + let url = poll_until( + "the CLI to open the authorization URL", + URL_TIMEOUT, + Duration::from_millis(100), + async || { + Ok(match tokio::fs::read_to_string(path).await { + Ok(contents) if !contents.trim().is_empty() => { + Poll::Ready(contents.trim().to_string()) + } + _ => Poll::Pending(format!("nothing in {}", path.display())), + }) + }, + ) + .await?; + Url::parse(&url) + .map_err(|error| Error::msg(format!("parse the authorization URL {url}: {error}"))) +} + +/// Request `url` with curl, keeping cookies in `jar`, and POST `form` if it +/// isn't empty. Returns the body. +async fn curl(options: &[&str], jar: &Path, url: &str, form: &[&str]) -> Result { + let mut command = Command::new("curl"); + command + .args([ + "--fail", + "--silent", + "--show-error", + "--max-time", + CURL_MAX_TIME, + ]) + .args(options) + .arg(jar) + .arg("--cookie") + .arg(jar); + for field in form { + command.args(["--data-urlencode", field]); + } + let output = command + .arg(url) + .kill_on_drop(true) + .output() + .await + .context("run curl")?; + testkit::ensure!( + output.status.success(), + "curl failed ({}): {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + ); + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// The URL Keycloak's login form posts to. +fn login_form_action(html: &str) -> Result { + let missing = || Error::msg(format!("no Keycloak login form in the page:\n{html}")); + let id = html.find("id=\"kc-form-login\"").ok_or_else(missing)?; + let start = html[..id].rfind("').ok_or_else(missing)?; + let tag = &html[start..end]; + let action = tag + .split_once("action=\"") + .and_then(|(_, rest)| rest.split_once('"')) + .map(|(action, _)| action) + .ok_or_else(missing)?; + Ok(action.replace("&", "&").replace("&", "&")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn finds_the_login_form_action() { + let html = r#"
"#; + assert_eq!( + login_form_action(html).unwrap(), + "http://kc/login?a=1&b=2&c=3" + ); + assert!(login_form_action("").is_err()); + } +} diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index 468cf29278..6ad295d790 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -15,9 +15,14 @@ //! | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `openshell-cli` | //! | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `admin@test` | //! | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `admin` | +//! | `OPENSHELL_TEST_KEYCLOAK_USER_USERNAME` | `user@test` | +//! | `OPENSHELL_TEST_KEYCLOAK_USER_PASSWORD` | `user` | +//! | `OPENSHELL_TEST_KEYCLOAK_USER_B_USERNAME` | `user-b@test` | +//! | `OPENSHELL_TEST_KEYCLOAK_USER_B_PASSWORD` | `user-b` | //! //! The issuer must be reachable from the test process. The client must allow -//! password grants. A missing variable fails the test; it never skips it. +//! password grants. The `USER` and `USER_B` variables name plain users, and +//! are read only by tests that sign in as them ([`User`]). A missing variable fails the test; it never skips it. //! The fixture waits for the realm to answer, since Keycloak may still be //! starting when the tests begin. //! @@ -33,6 +38,13 @@ //! } //! ``` //! +//! A suite whose gateway trusts the realm signs users in to it with +//! [`KeycloakContext::session`]: a [`Session`] holds an SDK client and a CLI +//! that send the user's token. [`browser_login`] runs the CLI's own browser +//! login once, with curl as the browser. The tmachine `-oidc` +//! environment (`tests/tmachine/ansible/playbooks/oidc.yaml`) runs such a +//! gateway. +//! //! The fixture owns no Keycloak state, so it has nothing to clean up. Tests //! that create resources in Keycloak or `OpenShell` must remove them, for //! example with [`Context::on_cleanup`]. @@ -47,10 +59,26 @@ use testkit::{Context, Error, Poll, Result, ResultExt as _, poll_until}; use tokio::io::AsyncWriteExt as _; use tokio::process::Command; +mod browser; +mod session; +mod token; + +pub use browser::{BrowserLogin, browser_login}; +pub use session::Session; +pub use token::{audience_contains, claims, has_realm_role, subject}; + pub const ISSUER_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_ISSUER"; pub const CLIENT_ID_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_CLIENT_ID"; pub const USERNAME_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_USERNAME"; pub const PASSWORD_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_PASSWORD"; +pub const USER_USERNAME_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_USER_USERNAME"; +pub const USER_PASSWORD_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_USER_PASSWORD"; +pub const USER_B_USERNAME_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_USER_B_USERNAME"; +pub const USER_B_PASSWORD_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_USER_B_PASSWORD"; +/// The scopes a login asks for so the gateway authorizes its token. The +/// `OpenShell` scopes are optional on the client, so Keycloak grants them +/// only when asked. +pub const OPENSHELL_SCOPES: &str = "openid profile email openshell:all"; /// How long Keycloak has to start answering, e.g. after the environment boots. // The wait runs inside the test, so it must leave room for the test body // within the runner's 2-minute per-test budget (`slow-timeout` in @@ -73,6 +101,29 @@ pub struct Keycloak { password: String, } +/// A user in the realm. [`User::Admin`] is the user the +/// `OPENSHELL_TEST_KEYCLOAK_USERNAME` variables name; the others are plain +/// users, read from their own variables only when a test logs in as them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum User { + /// A platform admin. + Admin, + /// A user with no platform role beyond signing in. + User, + /// A second plain user, for tests across two users' workspaces. + UserB, +} + +impl User { + fn env(self) -> (&'static str, &'static str) { + match self { + Self::Admin => (USERNAME_ENV, PASSWORD_ENV), + Self::User => (USER_USERNAME_ENV, USER_PASSWORD_ENV), + Self::UserB => (USER_B_USERNAME_ENV, USER_B_PASSWORD_ENV), + } + } +} + /// Tokens from one Keycloak login. #[derive(Clone)] pub struct Grant { @@ -175,15 +226,36 @@ impl Keycloak { /// Log in as the configured user with the password grant. pub async fn password_grant(&self) -> Result { + self.grant(&self.username, &self.password, "openid").await + } + + /// Log in as `user` with the password grant, asking for `scope`, such + /// as [`OPENSHELL_SCOPES`]. + pub async fn password_grant_as(&self, user: User, scope: &str) -> Result { + let (username, password) = self.credentials(user)?; + self.grant(&username, &password, scope).await + } + + /// The username and password of `user`. + pub fn credentials(&self, user: User) -> Result<(String, String)> { + if user == User::Admin { + return Ok((self.username.clone(), self.password.clone())); + } + let (username, password) = user.env(); + Ok((required(username)?, required(password)?)) + } + + async fn grant(&self, username: &str, password: &str, scope: &str) -> Result { let client_id = format!("client_id={}", self.client_id); - let username = format!("username={}", self.username); + let username_field = format!("username={username}"); + let scope = format!("scope={scope}"); let stdout = post_form( &self.token_endpoint(), - &["grant_type=password", &client_id, &username, "scope=openid"], - ("password", &self.password), + &["grant_type=password", &client_id, &username_field, &scope], + ("password", password), ) .await - .context("Keycloak password grant")?; + .context(format!("Keycloak password grant for {username}"))?; let response: Value = serde_json::from_slice(&stdout).context("decode Keycloak grant response")?; diff --git a/tests/support/keycloak/src/session.rs b/tests/support/keycloak/src/session.rs new file mode 100644 index 0000000000..d98eda01cb --- /dev/null +++ b/tests/support/keycloak/src/session.rs @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A user signed in to the gateway with a Keycloak token. + +use std::fmt; +use std::path::Path; + +use serde_json::json; +use tempfile::TempDir; +use testkit::sdk::OpenShellClient; +use testkit::{Cli, Context, Error, Result, ResultExt as _}; + +use crate::{Keycloak, KeycloakContext, OPENSHELL_SCOPES, User}; + +/// A realm user signed in to the test's gateway: an SDK client and a CLI +/// that send the user's access token. Both verify the gateway with its CA +/// and present no client certificate, so the gateway authorizes the user +/// alone. +/// +/// The token comes from the password grant and isn't refreshed. Keycloak's +/// access tokens outlast a test. +pub struct Session { + user: User, + username: String, + subject: String, + access_token: String, + client: OpenShellClient, + cli: Cli, + // The CLI's configuration, holding the token. Removed on drop. + _config_home: TempDir, +} + +impl KeycloakContext<'_> { + /// Sign `user` in to the test's gateway. See [`Session::login`]. + pub async fn session(&self, user: User) -> Result { + Session::login(self, self.keycloak(), user).await + } +} + +impl Session { + /// Log `user` in with the password grant and connect as them. + pub async fn login(cx: &Context, keycloak: &Keycloak, user: User) -> Result { + let (username, _) = keycloak.credentials(user)?; + let grant = keycloak.password_grant_as(user, OPENSHELL_SCOPES).await?; + let subject = crate::subject(&grant.access_token)?; + let base = cx.cli_handle(); + let gateway = base.gateway(); + let client = gateway + .connect_with_token(Some(&grant.access_token)) + .await + .context(format!("connect as {username}"))?; + let config_home = tempfile::tempdir().context("create a CLI config directory")?; + write_cli_session(config_home.path(), gateway, keycloak, &grant.access_token)?; + let cli = base.with_env("XDG_CONFIG_HOME", config_home.path()); + Ok(Self { + user, + username, + subject, + access_token: grant.access_token, + client, + cli, + _config_home: config_home, + }) + } + + pub fn user(&self) -> User { + self.user + } + + /// The Keycloak username, such as `user@test`. + pub fn username(&self) -> &str { + &self.username + } + + /// The token's subject, which names the user in workspace memberships. + pub fn subject(&self) -> &str { + &self.subject + } + + pub fn access_token(&self) -> &str { + &self.access_token + } + + /// An SDK client that sends the user's token. + pub fn client(&self) -> &OpenShellClient { + &self.client + } + + /// The CLI, signed in as the user, pointed at the test's workspace. + pub fn cli(&self) -> &Cli { + &self.cli + } + + /// The CLI, signed in as the user, pointed at `workspace`. + pub fn cli_in(&self, workspace: &str) -> Cli { + self.cli.clone().in_workspace(workspace) + } +} + +/// Write the CLI configuration `openshell gateway add --oidc-issuer` and a +/// login would: the registration, the token, and the gateway's CA, with no +/// client certificate. +fn write_cli_session( + config_home: &Path, + gateway: &testkit::Gateway, + keycloak: &Keycloak, + access_token: &str, +) -> Result { + let name = gateway.name.as_deref().ok_or_else(|| { + Error::msg("signing a user in to the CLI needs a registered gateway, not an endpoint") + })?; + let port = url::Url::parse(&gateway.endpoint) + .ok() + .and_then(|url| url.port_or_known_default()) + .ok_or_else(|| Error::msg(format!("no port in {}", gateway.endpoint)))?; + let dir = config_home.join("openshell").join("gateways").join(name); + std::fs::create_dir_all(&dir).context(format!("create {}", dir.display()))?; + let metadata = json!({ + "name": name, + "gateway_endpoint": gateway.endpoint, + "is_remote": false, + "gateway_port": port, + "auth_mode": "oidc", + "oidc_issuer": keycloak.issuer(), + "oidc_client_id": keycloak.client_id(), + "oidc_scopes": OPENSHELL_SCOPES, + }); + write(&dir.join("metadata.json"), &metadata.to_string())?; + let token = json!({ + "access_token": access_token, + "issuer": keycloak.issuer(), + "client_id": keycloak.client_id(), + }); + write(&dir.join("oidc_token.json"), &token.to_string())?; + if let Some(mtls) = gateway.mtls_dir() { + let target = dir.join("mtls"); + std::fs::create_dir_all(&target).context(format!("create {}", target.display()))?; + std::fs::copy(mtls.join("ca.crt"), target.join("ca.crt")) + .context(format!("copy the gateway CA from {}", mtls.display()))?; + } + Ok(()) +} + +fn write(path: &Path, contents: &str) -> Result { + std::fs::write(path, contents).context(format!("write {}", path.display())) +} + +impl fmt::Debug for Session { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Session") + .field("username", &self.username) + .field("subject", &self.subject) + .finish_non_exhaustive() + } +} diff --git a/tests/support/keycloak/src/token.rs b/tests/support/keycloak/src/token.rs new file mode 100644 index 0000000000..f0bdcd3634 --- /dev/null +++ b/tests/support/keycloak/src/token.rs @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reading the claims in a Keycloak access token. + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde_json::Value; +use testkit::{Error, Result, ResultExt as _}; + +/// The claims in `token`, a JWT. The signature isn't checked: the token +/// comes from the test's own login, and the gateway checks it. +pub fn claims(token: &str) -> Result { + let payload = token + .split('.') + .nth(1) + .ok_or_else(|| Error::msg("the access token is not a JWT"))?; + let bytes = URL_SAFE_NO_PAD + .decode(payload.trim_end_matches('=')) + .map_err(|error| Error::msg(format!("decode the access token's claims: {error}")))?; + serde_json::from_slice(&bytes).context("parse the access token's claims") +} + +/// The `sub` claim in `token`: the subject the gateway names the user by, +/// such as in workspace memberships. +pub fn subject(token: &str) -> Result { + claims(token)? + .get("sub") + .and_then(Value::as_str) + .filter(|subject| !subject.is_empty()) + .map(str::to_string) + .ok_or_else(|| Error::msg("the access token has no subject")) +} + +/// Whether `claims` lists `role` among its realm roles +/// (`realm_access.roles`). +pub fn has_realm_role(claims: &Value, role: &str) -> bool { + claims["realm_access"]["roles"] + .as_array() + .is_some_and(|roles| roles.iter().any(|r| r == role)) +} + +/// Whether the `aud` claim, a string or a list, includes `audience`. +pub fn audience_contains(claims: &Value, audience: &str) -> bool { + match &claims["aud"] { + Value::String(value) => value == audience, + Value::Array(values) => values.iter().any(|value| value == audience), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn jwt(claims: &Value) -> String { + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).unwrap()); + format!("e30.{payload}.sig") + } + + #[test] + fn reads_claims() { + let token = jwt(&serde_json::json!({ + "sub": "abc", + "aud": ["account", "openshell-cli"], + "realm_access": {"roles": ["openshell-user"]}, + })); + let claims = claims(&token).unwrap(); + assert_eq!(subject(&token).unwrap(), "abc"); + assert!(has_realm_role(&claims, "openshell-user")); + assert!(!has_realm_role(&claims, "openshell-admin")); + assert!(audience_contains(&claims, "openshell-cli")); + assert!(audience_contains( + &serde_json::json!({"aud": "openshell-cli"}), + "openshell-cli" + )); + } + + #[test] + fn rejects_a_token_without_a_subject() { + assert!(subject(&jwt(&serde_json::json!({"sub": ""}))).is_err()); + assert!(claims("not-a-jwt").is_err()); + } +} From 3ad35acae947819dad5e3552faaead948bd2c56d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:54:36 -0800 Subject: [PATCH 43/63] test(tmachine): add an OIDC gateway environment Signed-off-by: Matthew Grossman --- tests/support/README.md | 4 + tests/tmachine/ansible/playbooks/oidc.yaml | 24 ++++++ .../roles/oidc_gateway/defaults/main.yaml | 5 ++ .../files/operator-client-cert.sh | 47 +++++++++++ .../roles/oidc_gateway/tasks/main.yaml | 80 +++++++++++++++++++ tests/tmachine/config.nix | 12 +++ 6 files changed, 172 insertions(+) create mode 100644 tests/tmachine/ansible/playbooks/oidc.yaml create mode 100644 tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml create mode 100755 tests/tmachine/ansible/roles/oidc_gateway/files/operator-client-cert.sh create mode 100644 tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml diff --git a/tests/support/README.md b/tests/support/README.md index c13ba758e1..b355cf5b7a 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -23,6 +23,10 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_KEYCLOAK_USER_USERNAME` | `-oidc` (`playbooks/oidc.yaml`) | `keycloak-support` (`User::User`) | A realm user with no platform admin role, which the gateway signs in. | +| `OPENSHELL_TEST_KEYCLOAK_USER_PASSWORD` | `-oidc` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_KEYCLOAK_USER_B_USERNAME` | `-oidc` | `keycloak-support` (`User::UserB`) | A second such user, for tests across two users' workspaces. | +| `OPENSHELL_TEST_KEYCLOAK_USER_B_PASSWORD` | `-oidc` | `keycloak-support` | That user's password. | | `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | | `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | diff --git a/tests/tmachine/ansible/playbooks/oidc.yaml b/tests/tmachine/ansible/playbooks/oidc.yaml new file mode 100644 index 0000000000..bef53c3e2c --- /dev/null +++ b/tests/tmachine/ansible/playbooks/oidc.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sign gateway users in with the Keycloak realm keycloak.yaml runs, which +# runs first, and publish the realm's plain users for the oidc-auth suite. +--- +- name: OIDC gateway for tests + hosts: all + gather_facts: false + vars_files: + - ../roles/keycloak/defaults/main.yaml + roles: + - oidc_gateway + tasks: + # The values are the throwaway fixture realm's (scripts/keycloak-realm.json). + - name: Publish OIDC test variables + ansible.builtin.include_tasks: test-env.yaml + vars: + test_env_name: oidc + test_env_vars: + OPENSHELL_TEST_KEYCLOAK_USER_USERNAME: user@test + OPENSHELL_TEST_KEYCLOAK_USER_PASSWORD: user + OPENSHELL_TEST_KEYCLOAK_USER_B_USERNAME: user-b@test + OPENSHELL_TEST_KEYCLOAK_USER_B_PASSWORD: user-b diff --git a/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml b/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml new file mode 100644 index 0000000000..1e20e5a1da --- /dev/null +++ b/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +oidc_gateway_operator_script: /usr/local/libexec/openshell-test/operator-client-cert diff --git a/tests/tmachine/ansible/roles/oidc_gateway/files/operator-client-cert.sh b/tests/tmachine/ansible/roles/oidc_gateway/files/operator-client-cert.sh new file mode 100755 index 0000000000..72f39ff410 --- /dev/null +++ b/tests/tmachine/ansible/roles/oidc_gateway/files/operator-client-cert.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Make the packaged gateway's client certificate an operator's (OU=operator), +# so testkit, which connects with it, keeps full access when the gateway also +# accepts OIDC users. Runs as an ExecStartPre after `generate-certs`, which may +# have replaced the certificate, and does nothing if it is already an +# operator's. It then copies the certificate where the CLI finds it, since +# `generate-certs` copies its own there on every start. +set -eu + +tls_dir=${OPENSHELL_LOCAL_TLS_DIR:?OPENSHELL_LOCAL_TLS_DIR is not set} +ca_cert=$tls_dir/ca.crt +ca_key=$tls_dir/ca.key +cert=$tls_dir/client/tls.crt +key=$tls_dir/client/tls.key + +is_operator() { + openssl verify -CAfile "$ca_cert" "$cert" >/dev/null 2>&1 && + openssl x509 -in "$cert" -noout -subject -nameopt RFC2253 | grep -q 'OU=operator' +} + +if ! is_operator; then + work=$(mktemp -d) + trap 'rm -rf "$work"' EXIT + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$work/tls.key" + openssl req -new -key "$work/tls.key" \ + -subj "/O=openshell/OU=operator/CN=openshell-test-operator" -out "$work/tls.csr" + cat >"$work/ext.cnf" <<'EOF' +basicConstraints=CA:FALSE +keyUsage=critical,digitalSignature +extendedKeyUsage=clientAuth +EOF + openssl x509 -req -in "$work/tls.csr" -CA "$ca_cert" -CAkey "$ca_key" \ + -set_serial "0x$(openssl rand -hex 16)" -days 3650 \ + -extfile "$work/ext.cnf" -out "$work/tls.crt" + install -m 0600 "$work/tls.key" "$key" + install -m 0644 "$work/tls.crt" "$cert" + is_operator +fi + +cli_dir=${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/openshell/mtls +mkdir -p "$cli_dir" +install -m 0644 "$ca_cert" "$cli_dir/ca.crt" +install -m 0644 "$cert" "$cli_dir/tls.crt" +install -m 0600 "$key" "$cli_dir/tls.key" diff --git a/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml b/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml new file mode 100644 index 0000000000..1a7666c7ca --- /dev/null +++ b/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Make the packaged gateway trust the environment's Keycloak realm (the +# keycloak role) and keep testkit's mTLS connection working as an operator. +# +# With an OIDC issuer set, the gateway maps a client certificate to a user +# only when mTLS user authentication is on, and that user has no platform +# role. An operator certificate (OU=operator) has full access instead, so +# testkit's setup and cleanup work. The operator script re-signs the +# packaged gateway's client certificate on every start, after +# `generate-certs`, through a drop-in on the gateway's systemd user unit. +# Only the deb and rpm installers run that unit. +--- +- name: Detect the gateway user + ansible.builtin.include_role: + name: openshell_gateway + tasks_from: detect.yaml + +- name: Trust the Keycloak realm and operator certificates + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: oidc + gateway_overrides_content: | + [openshell.gateway.oidc] + issuer = "http://127.0.0.1:{{ keycloak_port }}/realms/{{ keycloak_realm }}" + dangerously_allow_insecure_http = true + audience = "openshell-cli" + jwks_ttl_secs = 60 + roles_claim = "realm_access.roles" + admin_role = "openshell-admin" + user_role = "openshell-user" + scopes_claim = "scope" + + [openshell.gateway.mtls_auth] + operator_enabled = true + +- name: Create the test scripts directory + become: true + ansible.builtin.file: + path: "{{ oidc_gateway_operator_script | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Install the operator certificate script + become: true + ansible.builtin.copy: + src: operator-client-cert.sh + dest: "{{ oidc_gateway_operator_script }}" + owner: root + group: root + mode: "0755" + +- name: Create the gateway unit's drop-in directory + become: true + ansible.builtin.file: + path: "{{ openshell_gateway_home }}/{{ item }}" + state: directory + owner: "{{ openshell_gateway_user }}" + group: "{{ openshell_gateway_user }}" + mode: "0755" + loop: + - .config + - .config/systemd + - .config/systemd/user + - .config/systemd/user/openshell-gateway.service.d + +- name: Run the operator certificate script before the gateway starts + become: true + ansible.builtin.copy: + dest: "{{ openshell_gateway_home }}/.config/systemd/user/openshell-gateway.service.d/oidc-operator.conf" + owner: "{{ openshell_gateway_user }}" + group: "{{ openshell_gateway_user }}" + mode: "0644" + content: | + [Service] + ExecStartPre={{ oidc_gateway_operator_script }} diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index e2fa50489f..bf9ccbfed9 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -109,6 +109,18 @@ let inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json"; }; }; + # Keycloak plus a gateway that signs users in with it, for the oidc-auth + # suite. Needs the deb or rpm installer, whose gateway runs mTLS. + oidc = { + bases = dockerBases; + setup = { + playbooks = [ + "ansible/playbooks/keycloak.yaml" + "ansible/playbooks/oidc.yaml" + ]; + inputs.keycloak_realm_file = "../../scripts/keycloak-realm.json"; + }; + }; # Hardware-free CDI fixtures, for the cdi suite. cdi = { bases = dockerBases ++ podmanBases; From 2a07cbfbb8186d2cd5fa254f7eb71101b806531b Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:54:36 -0800 Subject: [PATCH 44/63] docs(keycloak-support): note where the extra users come from Signed-off-by: Matthew Grossman --- tests/support/keycloak/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index 6ad295d790..205ecb8abc 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -22,7 +22,8 @@ //! //! The issuer must be reachable from the test process. The client must allow //! password grants. The `USER` and `USER_B` variables name plain users, and -//! are read only by tests that sign in as them ([`User`]). A missing variable fails the test; it never skips it. +//! are read only by tests that sign in as them ([`User`]). The tmachine +//! `-oidc` environment publishes them (`playbooks/oidc.yaml`). A missing variable fails the test; it never skips it. //! The fixture waits for the realm to answer, since Keycloak may still be //! starting when the tests begin. //! From 58680142db02438f2dc8074ec6418d57525320c1 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 17:06:48 -0800 Subject: [PATCH 45/63] feat(testkit): track a workspace's sandboxes, providers and profiles Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 3 + tests/testkit/testkit/src/context.rs | 6 +- tests/testkit/testkit/src/workspace.rs | 77 +++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index f98cf31245..bf9d958520 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -125,6 +125,9 @@ fails. A killed test process skips cleanup, so names are unique to the run. bearer token, such as an OIDC access token, and no client certificate. - `cx.workspace(suffix)` to create a workspace with cleanup. Its `add_member(subject, role)` and `remove_member(subject)` manage members. + `track_sandbox(name)` and `track_provider(name)` delete what another + identity creates in it; call them before the create. `import_profile(yaml)` + imports a provider profile into it. - `cx.on_cleanup(name, hook)` to remove something when the test ends. - `testkit::policy` for sandbox policies: `policy::base()` reads `testkit/src/base-policy.yaml`, which allows no network access, and diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index b12c55a420..761a3e4059 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -176,6 +176,10 @@ impl Context { &self.handles.grpc.client } + pub(crate) fn cleanup_list(&self) -> &Cleanup { + &self.handles.cleanup + } + /// A resource name unique to this run: `test--`. /// /// Sandbox names can't exceed [`MAX_SANDBOX_NAME_LEN`], so a sandbox @@ -345,7 +349,7 @@ fn single_compute_driver(info: &proto::GetGatewayInfoResponse) -> Result /// Register the cleanup hook that deletes sandbox `name`, unless it is /// already tracked. -fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) { +pub(crate) fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) { let client = client.clone(); let owned = name.to_string(); // The timeout wraps the whole hook, not just the wait, so a gateway that diff --git a/tests/testkit/testkit/src/workspace.rs b/tests/testkit/testkit/src/workspace.rs index 37263a6bf4..c7e5711dbc 100644 --- a/tests/testkit/testkit/src/workspace.rs +++ b/tests/testkit/testkit/src/workspace.rs @@ -9,8 +9,13 @@ use std::time::{Duration, Instant}; use openshell_sdk::raw::proto; use openshell_sdk::{DeleteOptions, OpenShellClient}; -use crate::context::Context; +use serde::Deserialize; + +use crate::cleanup::Cleanup; +use crate::cli::Cli; +use crate::context::{Context, track_sandbox}; use crate::error::{Error, Result, ResultExt as _}; +use crate::yaml_file::YamlFile; /// The longest workspace name the gateway accepts. const MAX_WORKSPACE_NAME_LEN: usize = 19; @@ -45,7 +50,12 @@ impl Context { .create_workspace(&name, HashMap::new()) .await .context(format!("create workspace {name}"))?; - Ok(Workspace { name, client }) + Ok(Workspace { + cli: self.cli_handle().in_workspace(&name), + name, + client, + cleanup: self.cleanup_list().clone(), + }) } } @@ -54,6 +64,8 @@ impl Context { pub struct Workspace { name: String, client: OpenShellClient, + cli: Cli, + cleanup: Cleanup, } impl Workspace { @@ -61,6 +73,67 @@ impl Workspace { &self.name } + /// Delete sandbox `name` in the workspace when the test ends, before the + /// workspace. Call it before the sandbox is created, by whoever creates + /// it, so cleanup deletes it even if the test fails first. + pub fn track_sandbox(&self, name: &str) { + track_sandbox(&self.cleanup, &self.client.workspace(&self.name), name); + } + + /// Delete provider `name` in the workspace when the test ends, before the + /// workspace. Call it before the provider is created. + pub fn track_provider(&self, name: &str) { + let client = self.client.clone(); + let workspace = self.name.clone(); + let owned = name.to_string(); + self.cleanup.push( + format!("delete provider {name} in workspace {workspace}"), + async move || { + client + .raw_grpc_fresh() + .await? + .delete_provider(proto::DeleteProviderRequest { + workspace_scope: Some(proto::workspace_selector(&workspace)), + name: owned, + allow_missing: true, + request_id: String::new(), + }) + .await?; + Ok(()) + }, + ); + } + + /// Import `profile`, a provider profile in YAML, into the workspace and + /// return its `id`. Cleanup deletes it after the workspace's providers. + pub async fn import_profile(&self, profile: &str) -> Result { + #[derive(Deserialize)] + struct Profile { + id: String, + } + let id = serde_yml::from_str::(profile) + .context("read the provider profile's id")? + .id; + // Deleting a missing profile succeeds, so register the delete before + // the import. + let cli = self.cli.clone(); + let owned = id.clone(); + self.cleanup.push( + format!("delete profile {id} in workspace {}", self.name), + async move || { + cli.run(["profile", "delete", &owned]).await?.success()?; + Ok(()) + }, + ); + let file = YamlFile::new(profile)?; + self.cli + .run(["profile", "import", "--file", file.path()?]) + .await? + .success() + .context(format!("import profile {id} into workspace {}", self.name))?; + Ok(id) + } + /// Make `subject`, an OIDC subject claim, a member with `role`. pub async fn add_member(&self, subject: &str, role: proto::WorkspaceRole) -> Result { self.client From af021e65f4975475722fa51b0ce0e8ba987a2fa5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 17:06:48 -0800 Subject: [PATCH 46/63] test(oidc-auth): port the OIDC authorization tests to testkit Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 1 + tests/suites/features/Cargo.lock | 13 + tests/suites/features/Cargo.toml | 2 +- tests/suites/features/oidc-auth/Cargo.toml | 18 + .../features/oidc-auth/tests/oidc_auth.rs | 1014 +++++++++++++++++ 5 files changed, 1047 insertions(+), 1 deletion(-) create mode 100644 tests/suites/features/oidc-auth/Cargo.toml create mode 100644 tests/suites/features/oidc-auth/tests/oidc_auth.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 951af177dd..fc2cde2e03 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -96,6 +96,7 @@ jobs: {"environment":"fedora-podman-rootful-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful-oidc","installer":"deb","family":"features","package":"oidc-auth","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index e6497fa45d..7fda327588 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1845,9 +1845,12 @@ dependencies = [ name = "keycloak-support" version = "0.0.0" dependencies = [ + "base64 0.22.1", "openshell-testkit", "serde_json", + "tempfile", "tokio", + "url", ] [[package]] @@ -2310,6 +2313,16 @@ dependencies = [ "asn1-rs", ] +[[package]] +name = "oidc-auth" +version = "0.0.0" +dependencies = [ + "keycloak-support", + "openshell-testkit", + "serde_json", + "url", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index d1592b7411..238dede742 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "routable-host-ip", "service-exposure"] +members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "oidc-auth", "routable-host-ip", "service-exposure"] diff --git a/tests/suites/features/oidc-auth/Cargo.toml b/tests/suites/features/oidc-auth/Cargo.toml new file mode 100644 index 0000000000..da7c0c8780 --- /dev/null +++ b/tests/suites/features/oidc-auth/Cargo.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "oidc-auth" +description = "Features suite: OIDC sign-in and workspace authorization against Keycloak" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +keycloak-support = { path = "../../../support/keycloak" } +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" +url = "2" diff --git a/tests/suites/features/oidc-auth/tests/oidc_auth.rs b/tests/suites/features/oidc-auth/tests/oidc_auth.rs new file mode 100644 index 0000000000..4433660168 --- /dev/null +++ b/tests/suites/features/oidc-auth/tests/oidc_auth.rs @@ -0,0 +1,1014 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! OIDC sign-in and workspace authorization against Keycloak. +//! +//! The gateway trusts the environment's Keycloak realm. Each test signs realm +//! users in with the password grant and checks what the gateway lets them +//! do, by gRPC status code and message. testkit's own connection is an mTLS +//! operator, which sets up and cleans up. The browser login runs once, and +//! a few checks run through the CLI, for what the CLI itself reports. + +use std::collections::HashMap; +use std::fmt; + +use keycloak_support::{BrowserLogin, KeycloakContext, OPENSHELL_SCOPES, Session, User}; +use serde_json::Value; +use testkit::sdk::raw::{AuthedGrpcClient, proto}; +use testkit::sdk::{DeleteOptions, ListOptions, OpenShellClient, SandboxSpec}; +use testkit::{ + Context, Credential, DELETE_TIMEOUT, Error, GrpcCode, Profile, Result, ResultExt as _, YamlFile, +}; +use url::Url; + +const ADMIN_ROLE_REQUIRED: &str = "role 'openshell-admin' required"; +const ONLY_PLATFORM_ADMINS: &str = "only platform admins can assign the workspace admin role"; +/// The provider credential the test profile defines. +const CREDENTIAL_ENV: &str = "API_KEY"; +const CREDENTIAL_VALUE: &str = "oidc-auth-test-value"; +/// A subject no realm user has. +const FAKE_SUBJECT: &str = "oidc-fake-member"; + +/// The browser (PKCE) login: the CLI opens Keycloak's authorization page +/// with a PKCE challenge, receives the code on its loopback callback, and +/// saves a session the gateway accepts. +#[testkit::test] +async fn pkce_browser_login_persists_session(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let login = keycloak_support::browser_login( + &kc, + kc.keycloak(), + User::User, + "profile email openshell:all", + ) + .await?; + check_authorization_url(&login.authorization_url, kc.keycloak().issuer())?; + let report = format!("{}{}", login.output.stdout, login.output.stderr); + testkit::ensure!( + report.contains("Authenticated successfully"), + "gateway add did not report the login:\n{report}" + ); + let subject = check_saved_session(&login, &kc)?; + + let whoami = login + .cli(&kc)? + .run(["whoami", "--output", "json"]) + .await? + .success()?; + let identity = json_object(&whoami.stdout)?; + testkit::ensure!( + identity["subject"] == subject.as_str(), + "whoami with the saved session reported {identity}, expected subject {subject}" + ); + Ok(()) +} + +/// `whoami` reports the identity the gateway validated from the token. +#[testkit::test] +async fn user_reports_gateway_validated_identity(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let user = kc.session(User::User).await?; + let output = user + .cli() + .run(["whoami", "--output", "json"]) + .await? + .success()?; + let identity = json_object(&output.stdout)?; + testkit::ensure!( + identity["subject"] == user.subject() + && identity["identity_provider"] == "oidc" + && identity["roles"] + .as_array() + .is_some_and(|roles| roles.iter().any(|role| role == "openshell-user")), + "whoami reported {identity}, expected subject {}, provider oidc and role openshell-user", + user.subject() + ); + Ok(()) +} + +/// A platform admin lists sandboxes and providers in a workspace it isn't a +/// member of, reads gateway info, manages a provider, and creates a +/// workspace. +#[testkit::test] +async fn platform_admin_smoke(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("adm").await?; + let profile = ws.import_profile(&test_profile(&kc.name("pf"))?).await?; + let admin = kc.session(User::Admin).await?; + let mut grpc = raw(&admin).await?; + + admin + .client() + .workspace(ws.name()) + .list_all_sandboxes(ListOptions::default()) + .await + .context("list sandboxes as platform admin")?; + grpc.list_providers(list_providers(ws.name())) + .await + .context("list providers as platform admin")?; + + let info = grpc + .get_gateway_info(proto::GetGatewayInfoRequest {}) + .await + .context("get gateway info as platform admin")? + .into_inner(); + let driver = kc.compute_driver().await?; + testkit::ensure!( + info.compute_drivers.iter().any(|d| d.name == driver), + "gateway info for the platform admin lists drivers {:?}, expected {driver}", + info.compute_drivers + .iter() + .map(|d| d.name.as_str()) + .collect::>() + ); + + let provider = kc.name("prov"); + ws.track_provider(&provider); + grpc.create_provider(provider_request(ws.name(), &provider, &profile)) + .await + .context("create a provider as platform admin")?; + let got = grpc + .get_provider(proto::GetProviderRequest { + workspace_scope: Some(proto::workspace_selector(ws.name())), + name: provider.clone(), + }) + .await + .context("get the provider as platform admin")? + .into_inner(); + let got_name = got + .provider + .and_then(|p| p.metadata) + .map(|m| m.name) + .unwrap_or_default(); + testkit::ensure!( + got_name == provider, + "provider get returned {got_name:?}, expected {provider}" + ); + grpc.delete_provider(delete_provider(ws.name(), &provider)) + .await + .context("delete the provider as platform admin")?; + + let created = kc.name("admw"); + delete_workspace_on_cleanup(&kc, admin.client(), &created); + admin + .client() + .create_workspace(&created, HashMap::new()) + .await + .context("create a workspace as platform admin")?; + let got = admin.client().get_workspace(&created).await?; + testkit::ensure!( + got.name == created, + "workspace get returned {}, expected {created}", + got.name + ); + Ok(()) +} + +/// Neither a plain user nor a workspace admin may read gateway info, or +/// create or delete workspaces. +#[testkit::test] +async fn non_admins_denied_platform_actions(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("pla").await?; + let admin = kc.session(User::Admin).await?; + let user = kc.session(User::User).await?; + let ws_admin = kc.session(User::UserB).await?; + ws.add_member(ws_admin.subject(), proto::WorkspaceRole::Admin) + .await?; + + for (who, session) in [("user", &user), ("workspace admin", &ws_admin)] { + let mut grpc = raw(session).await?; + expect_denied( + grpc.get_gateway_info(proto::GetGatewayInfoRequest {}).await, + ADMIN_ROLE_REQUIRED, + format!("get gateway info as {who}"), + )?; + let created = kc.name(if who == "user" { "nwu" } else { "nwa" }); + delete_workspace_on_cleanup(&kc, admin.client(), &created); + expect_denied( + session + .client() + .create_workspace(&created, HashMap::new()) + .await, + ADMIN_ROLE_REQUIRED, + format!("create a workspace as {who}"), + )?; + expect_denied( + session + .client() + .delete_workspace(ws.name(), DeleteOptions::default()) + .await, + ADMIN_ROLE_REQUIRED, + format!("delete workspace {} as {who}", ws.name()), + )?; + } + admin.client().get_workspace(ws.name()).await?; + Ok(()) +} + +/// Members, as users or admins, read their workspace, its members, its +/// sandboxes and its providers, and see it in their workspace list. +#[testkit::test] +async fn workspace_user_read_access(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let user = kc.session(User::User).await?; + for (suffix, role) in [ + ("rdu", proto::WorkspaceRole::User), + ("rda", proto::WorkspaceRole::Admin), + ] { + let ws = kc.workspace(suffix).await?; + ws.add_member(user.subject(), role).await?; + let action = |what: &str| format!("{what} as a workspace {}", role.as_str_name()); + + let got = user + .client() + .get_workspace(ws.name()) + .await + .context(action("get the workspace"))?; + testkit::ensure!(got.name == ws.name(), "workspace get returned {}", got.name); + let listed = user + .client() + .list_all_workspaces(ListOptions::default()) + .await + .context(action("list workspaces"))?; + testkit::ensure!( + listed.iter().any(|w| w.name == ws.name()), + "{} is missing from the member's workspace list", + ws.name() + ); + let members = members(&mut raw(&user).await?, ws.name()) + .await + .context(action("list members"))?; + testkit::ensure!( + members.iter().any(|m| m == user.subject()), + "the member list of {} is missing {}: {members:?}", + ws.name(), + user.subject() + ); + user.client() + .workspace(ws.name()) + .list_all_sandboxes(ListOptions::default()) + .await + .context(action("list sandboxes"))?; + raw(&user) + .await? + .list_providers(list_providers(ws.name())) + .await + .context(action("list providers"))?; + } + Ok(()) +} + +/// A platform admin and a workspace user each create a sandbox in the +/// workspace, see it listed, and delete it. +#[testkit::test] +async fn sandbox_create_delete_by_role(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("sbx").await?; + let admin = kc.session(User::Admin).await?; + let user = kc.session(User::User).await?; + ws.add_member(user.subject(), proto::WorkspaceRole::User) + .await?; + + for (suffix, session) in [("sba", &admin), ("sbu", &user)] { + let name = kc.name(suffix); + let client = session.client().workspace(ws.name()); + let who = session.username(); + ws.track_sandbox(&name); + // Keep the sandbox running, so this checks authorization rather + // than how fast a finished command's sandbox goes. + client + .create_sandbox(SandboxSpec { + name: Some(name.clone()), + command: vec!["sleep".to_string(), "infinity".to_string()], + ..SandboxSpec::default() + }) + .await + .context(format!("create sandbox {name} as {who}"))?; + let listed = sandbox_names(&client.list_all_sandboxes(ListOptions::default()).await?); + testkit::ensure!( + listed.contains(&name), + "{who}'s sandbox list is missing {name}: {listed:?}" + ); + client + .delete_sandbox(&name, DeleteOptions::default()) + .await + .context(format!("delete sandbox {name} as {who}"))?; + client.wait_deleted(&name, DELETE_TIMEOUT, None).await?; + let listed = sandbox_names(&client.list_all_sandboxes(ListOptions::default()).await?); + testkit::ensure!( + !listed.contains(&name), + "{name} is still listed after {who} deleted it: {listed:?}" + ); + } + Ok(()) +} + +/// A workspace user may not create or delete providers, or manage members. +/// The CLI tells them how to get the admin role. +#[testkit::test] +async fn workspace_user_denied_management(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("wum").await?; + let profile = ws.import_profile(&test_profile(&kc.name("pf"))?).await?; + let admin = kc.session(User::Admin).await?; + let user = kc.session(User::User).await?; + ws.add_member(user.subject(), proto::WorkspaceRole::User) + .await?; + let target = kc.name("tgt"); + ws.track_provider(&target); + raw(&admin) + .await? + .create_provider(provider_request(ws.name(), &target, &profile)) + .await + .context("create the provider to delete")?; + + let needs_admin = format!( + "workspace role 'admin' required in workspace '{}'", + ws.name() + ); + let mut grpc = raw(&user).await?; + let created = kc.name("new"); + ws.track_provider(&created); + expect_denied( + grpc.create_provider(provider_request(ws.name(), &created, &profile)) + .await, + &needs_admin, + "create a provider as a workspace user", + )?; + expect_denied( + grpc.delete_provider(delete_provider(ws.name(), &target)) + .await, + &needs_admin, + "delete a provider as a workspace user", + )?; + expect_denied( + grpc.add_workspace_member(add_member( + ws.name(), + FAKE_SUBJECT, + proto::WorkspaceRole::User, + )) + .await, + &needs_admin, + "add a member as a workspace user", + )?; + raw(&admin) + .await? + .get_provider(proto::GetProviderRequest { + workspace_scope: Some(proto::workspace_selector(ws.name())), + name: target.clone(), + }) + .await + .context("the denied delete left the provider")?; + + let denied = user + .cli_in(ws.name()) + .run([ + "workspace", + "member", + "add", + "--workspace", + ws.name(), + "--subject", + FAKE_SUBJECT, + "--role", + "user", + ]) + .await? + .failure()?; + let report = compact(&denied.report_text()); + let remediation = compact(&format!( + "openshell workspace member add --workspace '{}' --subject '{}' --role admin", + ws.name(), + user.subject() + )); + testkit::ensure!( + report.contains(&compact("workspace role 'admin'")) && report.contains(&remediation), + "the denial does not tell the user how to get the admin role:\n{}", + denied.report_text() + ); + Ok(()) +} + +/// A workspace admin creates and deletes providers and adds and removes +/// members. A removed member, by the workspace admin or a platform admin, +/// loses access. +#[testkit::test] +async fn workspace_admin_manages_providers_and_members(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("wam").await?; + let profile = ws.import_profile(&test_profile(&kc.name("pf"))?).await?; + let admin = kc.session(User::Admin).await?; + let ws_admin = kc.session(User::User).await?; + let member = kc.session(User::UserB).await?; + ws.add_member(ws_admin.subject(), proto::WorkspaceRole::Admin) + .await?; + let mut grpc = raw(&ws_admin).await?; + + let created = kc.name("own"); + ws.track_provider(&created); + grpc.create_provider(provider_request(ws.name(), &created, &profile)) + .await + .context("create a provider as workspace admin")?; + let target = kc.name("tgt"); + ws.track_provider(&target); + raw(&admin) + .await? + .create_provider(provider_request(ws.name(), &target, &profile)) + .await + .context("create the provider to delete")?; + grpc.delete_provider(delete_provider(ws.name(), &target)) + .await + .context("delete a provider as workspace admin")?; + + grpc.add_workspace_member(add_member( + ws.name(), + member.subject(), + proto::WorkspaceRole::User, + )) + .await + .context("add a member as workspace admin")?; + let listed = members(&mut grpc, ws.name()).await?; + testkit::ensure!( + listed.iter().any(|m| m == member.subject()), + "the added member is not listed: {listed:?}" + ); + member + .client() + .get_workspace(ws.name()) + .await + .context("get the workspace as the added member")?; + + let not_member = format!("not a member of workspace '{}'", ws.name()); + grpc.remove_workspace_member(remove_member(ws.name(), member.subject())) + .await + .context("remove a member as workspace admin")?; + expect_denied( + member.client().get_workspace(ws.name()).await, + ¬_member, + "get the workspace after the workspace admin removed the member", + )?; + + raw(&admin) + .await? + .add_workspace_member(add_member( + ws.name(), + member.subject(), + proto::WorkspaceRole::User, + )) + .await + .context("add the member back as platform admin")?; + raw(&admin) + .await? + .remove_workspace_member(remove_member(ws.name(), member.subject())) + .await + .context("remove the member as platform admin")?; + expect_denied( + member.client().get_workspace(ws.name()).await, + ¬_member, + "get the workspace after a platform admin removed the member", + )?; + Ok(()) +} + +/// Only a platform admin may make a member a workspace admin. +#[testkit::test] +async fn workspace_admin_cannot_grant_admin(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("wag").await?; + let ws_admin = kc.session(User::User).await?; + ws.add_member(ws_admin.subject(), proto::WorkspaceRole::Admin) + .await?; + let mut grpc = raw(&ws_admin).await?; + expect_denied( + grpc.add_workspace_member(add_member( + ws.name(), + FAKE_SUBJECT, + proto::WorkspaceRole::Admin, + )) + .await, + ONLY_PLATFORM_ADMINS, + "grant the workspace admin role as workspace admin", + )?; + let listed = members(&mut grpc, ws.name()).await?; + testkit::ensure!( + !listed.iter().any(|m| m == FAKE_SUBJECT), + "the denied grant added a member: {listed:?}" + ); + Ok(()) +} + +/// Being an admin or a user of one workspace gives no access to another. +#[testkit::test] +async fn cross_workspace_access_denied(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws_a = kc.workspace("xa").await?; + let ws_b = kc.workspace("xb").await?; + let admin = kc.session(User::Admin).await?; + let user = kc.session(User::User).await?; + let user_b = kc.session(User::UserB).await?; + ws_b.add_member(user_b.subject(), proto::WorkspaceRole::User) + .await?; + let b = ws_b.name(); + let not_member = format!("not a member of workspace '{b}'"); + + for (suffix, role) in [ + ("xsa", proto::WorkspaceRole::Admin), + ("xsu", proto::WorkspaceRole::User), + ] { + ws_a.add_member(user.subject(), role).await?; + let who = format!("a workspace {} of {}", role.as_str_name(), ws_a.name()); + let mut grpc = raw(&user).await?; + let client = user.client(); + + expect_denied( + client.get_workspace(b).await, + ¬_member, + format!("get {b} as {who}"), + )?; + expect_denied( + grpc.list_workspace_members(list_members(b)).await, + ¬_member, + format!("list the members of {b} as {who}"), + )?; + expect_denied( + grpc.add_workspace_member(add_member(b, FAKE_SUBJECT, proto::WorkspaceRole::User)) + .await, + ¬_member, + format!("add a member to {b} as {who}"), + )?; + expect_denied( + client + .workspace(b) + .list_all_sandboxes(ListOptions::default()) + .await, + ¬_member, + format!("list the sandboxes in {b} as {who}"), + )?; + expect_denied( + grpc.list_providers(list_providers(b)).await, + ¬_member, + format!("list the providers in {b} as {who}"), + )?; + let sandbox = kc.name(suffix); + ws_b.track_sandbox(&sandbox); + expect_denied( + client + .workspace(b) + .create_sandbox(SandboxSpec { + name: Some(sandbox.clone()), + command: vec!["sleep".to_string(), "infinity".to_string()], + ..SandboxSpec::default() + }) + .await, + ¬_member, + format!("create a sandbox in {b} as {who}"), + )?; + ws_a.remove_member(user.subject()).await?; + } + + let listed = sandbox_names( + &admin + .client() + .workspace(b) + .list_all_sandboxes(ListOptions::default()) + .await?, + ); + testkit::ensure!( + listed.is_empty(), + "the denied creates left sandboxes in {b}: {listed:?}" + ); + Ok(()) +} + +/// A user's workspace list shows their own workspace and hides another +/// user's and `default`. +#[testkit::test] +async fn workspace_list_hides_other_workspaces(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws_a = kc.workspace("la").await?; + let ws_b = kc.workspace("lb").await?; + let user = kc.session(User::User).await?; + let user_b = kc.session(User::UserB).await?; + ws_a.add_member(user.subject(), proto::WorkspaceRole::User) + .await?; + ws_b.add_member(user_b.subject(), proto::WorkspaceRole::User) + .await?; + + let names: Vec = user + .client() + .list_all_workspaces(ListOptions::default()) + .await + .context("list workspaces as a user")? + .into_iter() + .map(|w| w.name) + .collect(); + testkit::ensure!( + names.iter().any(|n| n == ws_a.name()), + "the user's workspace list is missing {}: {names:?}", + ws_a.name() + ); + testkit::ensure!( + !names.iter().any(|n| n == ws_b.name() || n == "default"), + "the user's workspace list shows a workspace they aren't a member of: {names:?}" + ); + Ok(()) +} + +/// A workspace admin can't create a provider in another workspace. The CLI +/// reports a permission denial without the credential, and the workspace +/// gets no provider. +#[testkit::test] +async fn workspace_admin_cannot_manage_another_workspace_providers(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws_a = kc.workspace("pa").await?; + let ws_b = kc.workspace("pb").await?; + let admin = kc.session(User::Admin).await?; + let ws_admin = kc.session(User::User).await?; + let user_b = kc.session(User::UserB).await?; + ws_a.add_member(ws_admin.subject(), proto::WorkspaceRole::Admin) + .await?; + ws_b.add_member(user_b.subject(), proto::WorkspaceRole::User) + .await?; + let provider = kc.name("xprov"); + ws_b.track_provider(&provider); + + let credential = format!("OPENAI_API_KEY={CREDENTIAL_VALUE}"); + let denied = ws_admin + .cli_in(ws_b.name()) + .run([ + "provider", + "create", + "--name", + &provider, + "--type", + "openai", + "--credential", + &credential, + ]) + .await? + .failure()?; + // The profile lookup redacts the gateway's membership details, so check + // the code and the guidance. + let report = compact(&denied.report_text()); + testkit::ensure!( + report.contains("PERMISSION_DENIED") + && report.contains("verifyworkspacemembershipandrequiredpermissions"), + "the cross-workspace provider create did not report a permission denial:\n{}", + denied.report_text() + ); + testkit::ensure!( + !denied.stdout.contains(CREDENTIAL_VALUE) && !denied.stderr.contains(CREDENTIAL_VALUE), + "the denial printed the credential value" + ); + + // Ask as another identity, so a provider created before the denial + // can't hide behind the failed command. + let listing = raw(&admin) + .await? + .list_providers(list_providers(ws_b.name())) + .await + .context("list providers as platform admin")? + .into_inner(); + testkit::ensure!( + listing.providers.is_empty() && listing.next_page_token.is_empty(), + "the denied create left providers in {}: {:?}", + ws_b.name(), + listing + .providers + .iter() + .filter_map(|p| p.metadata.as_ref().map(|m| m.name.as_str())) + .collect::>() + ); + Ok(()) +} + +/// A workspace user naming a platform provider profile with `--provider` +/// gets past the profile lookup without a platform-admin check. +#[testkit::test] +async fn user_can_resolve_provider_profile_for_sandbox(cx: &mut Context) -> Result { + let kc = setup(cx).await?; + let ws = kc.workspace("rp").await?; + let user = kc.session(User::User).await?; + ws.add_member(user.subject(), proto::WorkspaceRole::User) + .await?; + let profile = import_platform_profile(&kc, &kc.name("gpf")).await?; + let sandbox = kc.name("rps"); + ws.track_sandbox(&sandbox); + + // No provider of the profile's type exists, and without a terminal the + // CLI can't offer to create one, so it stops there. That error means + // the lookup worked; a permission error would mean the user was kept + // out of the catalog. + let output = user + .cli_in(ws.name()) + .run([ + "sandbox", + "create", + "--name", + &sandbox, + "--no-tty", + "--provider", + &profile, + ]) + .await?; + let report = output.report_text(); + testkit::ensure!( + !report.to_ascii_lowercase().contains("platform admin"), + "the workspace user hit a platform-admin check resolving the profile:\n{report}" + ); + testkit::ensure!( + report.contains("missing required provider"), + "expected the missing-provider error, got:\n{report}" + ); + Ok(()) +} + +// ── Helpers ──────────────────────────────────────────────────────────── + +/// Read the Keycloak configuration and check testkit connects as an +/// operator, which the tests' setup and cleanup need. +async fn setup(cx: &mut Context) -> Result> { + let kc = KeycloakContext::new(cx).await?; + let operator = kc.cli_handle().gateway().connect().await?; + match operator + .raw_grpc_fresh() + .await? + .get_current_user(proto::GetCurrentUserRequest {}) + .await + { + Err(status) if status.code() == GrpcCode::PermissionDenied => Ok(kc), + Err(status) => Err(Error::from(status).context("check testkit connects as an operator")), + Ok(user) => Err(Error::msg(format!( + "testkit's client certificate signs in as user {:?} ({}), not as an operator; \ + the -oidc environment makes it an operator's (roles/oidc_gateway)", + user.get_ref().subject, + user.get_ref().identity_provider + ))), + } +} + +async fn raw(session: &Session) -> Result { + Ok(session.client().raw_grpc_fresh().await?) +} + +/// Fail unless `result` is a permission denial whose message contains +/// `message`. +fn expect_denied>( + result: std::result::Result, + message: &str, + action: impl fmt::Display, +) -> Result { + let error = match result { + Ok(_) => return Err(Error::msg(format!("{action}: allowed, expected a denial"))), + Err(error) => error.into(), + }; + testkit::ensure!( + error.grpc_code() == Some(GrpcCode::PermissionDenied) + && error.to_string().contains(message), + "{action}: expected PERMISSION_DENIED with {message:?}, got {:?}: {error}", + error.grpc_code() + ); + Ok(()) +} + +/// A profile whose providers need one credential. +fn test_profile(id: &str) -> Result { + Profile::new(id) + .credential( + Credential::new("api_key") + .env_var(CREDENTIAL_ENV) + .required(), + ) + .to_yaml() +} + +/// Import a profile at platform scope as the operator, with cleanup. +async fn import_platform_profile(cx: &Context, id: &str) -> Result { + let cli = cx.cli_handle(); + let owned = id.to_string(); + cx.on_cleanup(format!("delete platform profile {id}"), async move || { + cli.run(["profile", "delete", "--global", &owned]) + .await? + .success()?; + Ok(()) + }); + let file = YamlFile::new(&test_profile(id)?)?; + cx.cli(["profile", "import", "--global", "--file", file.path()?]) + .await? + .success() + .context(format!("import platform profile {id}"))?; + Ok(id.to_string()) +} + +/// Delete workspace `name`, which a test asks the gateway to create, when +/// the test ends, as `admin`. +fn delete_workspace_on_cleanup(cx: &Context, admin: &OpenShellClient, name: &str) { + let admin = admin.clone(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete workspace {name}"), async move || { + admin + .delete_workspace( + &owned, + DeleteOptions { + allow_missing: true, + }, + ) + .await?; + Ok(()) + }); +} + +fn provider_request(workspace: &str, name: &str, profile: &str) -> proto::CreateProviderRequest { + proto::CreateProviderRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + provider: Some(proto::Provider { + metadata: Some(proto::ObjectMeta { + name: name.to_string(), + workspace: workspace.to_string(), + ..Default::default() + }), + r#type: profile.to_string(), + credentials: [(CREDENTIAL_ENV.to_string(), CREDENTIAL_VALUE.to_string())].into(), + ..Default::default() + }), + ..Default::default() + } +} + +fn delete_provider(workspace: &str, name: &str) -> proto::DeleteProviderRequest { + proto::DeleteProviderRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + name: name.to_string(), + ..Default::default() + } +} + +fn list_providers(workspace: &str) -> proto::ListProvidersRequest { + proto::ListProvidersRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + ..Default::default() + } +} + +fn add_member( + workspace: &str, + subject: &str, + role: proto::WorkspaceRole, +) -> proto::AddWorkspaceMemberRequest { + proto::AddWorkspaceMemberRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + principal_subject: subject.to_string(), + role: role.into(), + ..Default::default() + } +} + +fn remove_member(workspace: &str, subject: &str) -> proto::RemoveWorkspaceMemberRequest { + proto::RemoveWorkspaceMemberRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + principal_subject: subject.to_string(), + ..Default::default() + } +} + +fn list_members(workspace: &str) -> proto::ListWorkspaceMembersRequest { + proto::ListWorkspaceMembersRequest { + workspace_scope: Some(proto::workspace_selector(workspace)), + ..Default::default() + } +} + +/// The subjects of `workspace`'s members. The test workspaces have a few +/// members, so the first page holds them all. +async fn members(grpc: &mut AuthedGrpcClient, workspace: &str) -> Result> { + let response = grpc + .list_workspace_members(list_members(workspace)) + .await + .context(format!("list the members of {workspace}"))? + .into_inner(); + testkit::ensure!( + response.next_page_token.is_empty(), + "the members of {workspace} span more than one page" + ); + Ok(response + .members + .into_iter() + .map(|m| m.principal_subject) + .collect()) +} + +fn sandbox_names(sandboxes: &[testkit::sdk::SandboxRef]) -> Vec { + sandboxes.iter().map(|s| s.name.clone()).collect() +} + +/// `text` without whitespace, for matching an error report the CLI may wrap +/// inside a word, such as a quoted command. +fn compact(text: &str) -> String { + text.chars().filter(|c| !c.is_whitespace()).collect() +} + +/// The JSON object in `stdout`. The CLI may print startup notices before +/// it. +fn json_object(stdout: &str) -> Result { + let start = stdout.find('{'); + let end = stdout.rfind('}'); + let (Some(start), Some(end)) = (start, end) else { + return Err(Error::msg(format!( + "no JSON object in the output:\n{stdout}" + ))); + }; + serde_json::from_str(&stdout[start..=end]).context("parse the CLI's JSON output") +} + +/// Check the URL the CLI opened asks Keycloak for a PKCE (S256) code with +/// the requested scopes, returned to the CLI's loopback callback. +fn check_authorization_url(url: &Url, issuer: &str) -> Result { + let issuer = Url::parse(issuer).map_err(|e| Error::msg(format!("parse {issuer}: {e}")))?; + let expected_path = format!( + "{}/protocol/openid-connect/auth", + issuer.path().trim_end_matches('/') + ); + testkit::ensure!( + url.path() == expected_path, + "authorization URL {url} is not {expected_path}" + ); + let params: HashMap = url.query_pairs().into_owned().collect(); + let param = |name: &str| params.get(name).map(String::as_str).unwrap_or_default(); + testkit::ensure!( + param("response_type") == "code" + && param("client_id") == "openshell-cli" + && param("code_challenge_method") == "S256", + "authorization URL {url} does not ask for a PKCE code for openshell-cli" + ); + let challenge = param("code_challenge"); + testkit::ensure!( + challenge.len() == 43 + && challenge + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')), + "PKCE challenge {challenge:?} is not an unpadded base64url SHA-256" + ); + testkit::ensure!( + !param("state").is_empty(), + "authorization URL {url} has no state" + ); + let scopes: Vec<&str> = param("scope").split_whitespace().collect(); + for scope in OPENSHELL_SCOPES.split_whitespace() { + testkit::ensure!( + scopes.contains(&scope), + "authorization URL scopes {scopes:?} lack {scope}" + ); + } + let redirect = Url::parse(param("redirect_uri")) + .map_err(|e| Error::msg(format!("parse the redirect URI in {url}: {e}")))?; + testkit::ensure!( + redirect.scheme() == "http" + && redirect.host_str() == Some("127.0.0.1") + && redirect.port().is_some() + && redirect.path() == "/callback", + "redirect URI {redirect} is not the CLI's loopback callback" + ); + Ok(()) +} + +/// Check the CLI saved the OIDC registration and the user's tokens, and +/// return the token's subject. +fn check_saved_session(login: &BrowserLogin, kc: &KeycloakContext<'_>) -> Result { + let keycloak = kc.keycloak(); + let dir = login.gateway_dir(); + let metadata = read_json(&dir.join("metadata.json"))?; + testkit::ensure!( + metadata["auth_mode"] == "oidc" + && metadata["oidc_issuer"] == keycloak.issuer() + && metadata["oidc_client_id"] == keycloak.client_id() + && metadata["oidc_scopes"] == "profile email openshell:all", + "the saved registration is not the OIDC one: {metadata}" + ); + let token = read_json(&dir.join("oidc_token.json"))?; + let access_token = token["access_token"].as_str().unwrap_or_default(); + testkit::ensure!( + !access_token.is_empty() + && token["refresh_token"] + .as_str() + .is_some_and(|t| !t.is_empty()) + && token["issuer"] == keycloak.issuer() + && token["client_id"] == keycloak.client_id(), + "the saved token file lacks the access token, refresh token, issuer or client" + ); + let (username, _) = keycloak.credentials(User::User)?; + let claims = keycloak_support::claims(access_token)?; + testkit::ensure!( + keycloak_support::audience_contains(&claims, "openshell-cli") + && claims["azp"] == "openshell-cli" + && claims["preferred_username"] == username.as_str() + && keycloak_support::has_realm_role(&claims, "openshell-user"), + "the saved access token's claims are not the user's: {claims}" + ); + keycloak_support::subject(access_token) +} + +fn read_json(path: &std::path::Path) -> Result { + let text = std::fs::read_to_string(path).context(format!("read {}", path.display()))?; + serde_json::from_str(&text).context(format!("parse {}", path.display())) +} From ab47b4c085c360c5b09a87a9f2e4f8857551eebc Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 17:08:10 -0800 Subject: [PATCH 47/63] test(e2e): drop the OIDC tests the oidc-auth suite now covers Signed-off-by: Matthew Grossman --- e2e/rust/tests/oidc_pkce.rs | 1134 +++-------------------------------- 1 file changed, 67 insertions(+), 1067 deletions(-) diff --git a/e2e/rust/tests/oidc_pkce.rs b/e2e/rust/tests/oidc_pkce.rs index 409d99bf30..461db3b287 100644 --- a/e2e/rust/tests/oidc_pkce.rs +++ b/e2e/rust/tests/oidc_pkce.rs @@ -10,912 +10,100 @@ //! callback and token exchange used by a real browser without requiring a GUI. //! It logs in as the fixture identities and verifies standard-user and admin-only //! actions against a live Docker- or Podman-backed gateway. +//! +//! The rest of this coverage lives in the `oidc-auth` testkit suite. These +//! four checks duplicate it and are not ported. -use std::collections::HashMap; -use std::ffi::OsString; -use std::fs::Permissions; -use std::os::unix::fs::PermissionsExt; -use std::path::Path; -use std::process::{Output, Stdio}; -use std::time::{Duration, Instant}; - -use base64::Engine as _; -use openshell_e2e::harness::binary::openshell_cmd; -use serde_json::Value; -use serial_test::serial; -use tokio::process::Command; -use tokio::sync::Mutex; -use url::Url; - -static SANDBOX_LIFECYCLE_LOCK: Mutex<()> = Mutex::const_new(()); - -#[derive(Clone, Copy)] -struct IdentityScenario { - gateway_name: &'static str, - username: &'static str, - password: &'static str, - expected_role: &'static str, -} - -const ADMIN: IdentityScenario = IdentityScenario { - gateway_name: "oidc-pkce-admin", - username: "admin@test", - password: "admin", - expected_role: "openshell-admin", -}; - -const USER: IdentityScenario = IdentityScenario { - gateway_name: "oidc-pkce-user", - username: "user@test", - password: "user", - expected_role: "openshell-user", -}; - -const USER_B: IdentityScenario = IdentityScenario { - gateway_name: "oidc-pkce-user-b", - username: "user-b@test", - password: "user-b", - expected_role: "openshell-user", -}; - -struct LoginSession { - config_home: tempfile::TempDir, - identity: IdentityScenario, - subject: String, -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_list_sandboxes() { - let session = login_identity(ADMIN).await; - assert_allowed( - &session, - &["sandbox", "list", "--output", "json"], - "list sandboxes", - ) - .await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_report_gateway_validated_identity() { - let session = login_identity(USER).await; - let output = assert_allowed( - &session, - &["whoami", "--output", "json"], - "report current identity", - ) - .await; - let stdout = String::from_utf8(output.stdout).expect("whoami output should be UTF-8"); - let json_start = stdout.find('{').expect("whoami output should contain JSON"); - let json_end = stdout - .rfind('}') - .expect("whoami output should contain a complete JSON object"); - let identity: Value = serde_json::from_str(&stdout[json_start..=json_end]) - .expect("whoami --output json should return JSON on stdout"); - - assert_eq!(identity["subject"], session.subject); - assert_eq!(identity["identity_provider"], "oidc"); - assert!( - identity["roles"] - .as_array() - .is_some_and(|roles| roles.iter().any(|role| role == USER.expected_role)), - "whoami should report the configured user role: {identity}" - ); -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_list_sandboxes() { - const WORKSPACE: &str = "oidc-user-list-sb"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - assert_workspace_allowed( - &user, - WORKSPACE, - &["sandbox", "list", "--output", "json"], - "list sandboxes", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_create_sandbox() { - let session = login_identity(ADMIN).await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_create_sandbox(&session, "default", "oidc-admin-create").await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_create_sandbox() { - const WORKSPACE: &str = "oidc-user-create"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_create_sandbox(&user, WORKSPACE, "oidc-user-create").await; - delete_workspace(&admin, WORKSPACE).await; -} - -/// Workspace users must be able to resolve a provider profile when naming a -/// provider with `--provider`, without requiring Platform Admin access. -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_resolve_provider_profile_for_sandbox() { - const WORKSPACE: &str = "oidc-named-provider"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - - // `claude-code` names no existing provider, so the CLI has to look up the - // profile of that id before it can auto-create one. The lane imported the - // example profiles at platform scope; reaching them from a workspace is - // what this test guards. - let output = run_workspace_cli( - &user, - WORKSPACE, - &[ - "sandbox", - "create", - "--name", - "oidc-named-provider", - "--no-tty", - "--provider", - "claude-code", - ], - ) - .await; - let combined = combined_output(&output); - - // The provider cannot be auto-created without a terminal to confirm at, - // so creation stops there. That error proves the profile lookup - // succeeded; a permission error would mean the workspace user was gated - // out of the catalog. - assert!( - !combined.to_ascii_lowercase().contains("platform admin"), - "workspace user hit a platform-admin gate while resolving a provider profile:\n{combined}" - ); - assert!( - combined.contains("missing required provider"), - "expected missing-provider error for non-interactive session, got:\n{combined}" - ); - - let _ = run_workspace_cli( - &user, - WORKSPACE, - &["sandbox", "delete", "oidc-named-provider"], - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_delete_sandbox() { - let session = login_identity(ADMIN).await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_delete_sandbox(&session, "default", "oidc-admin-delete").await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_delete_sandbox() { - const WORKSPACE: &str = "oidc-user-delete"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_delete_sandbox(&user, WORKSPACE, "oidc-user-delete").await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_inspect_gateway() { - let session = login_identity(ADMIN).await; - let output = assert_allowed(&session, &["gateway", "info"], "inspect gateway info").await; - let info = combined_output(&output); - let expected_driver = - std::env::var("OPENSHELL_E2E_DRIVER").expect("OIDC E2E requires OPENSHELL_E2E_DRIVER"); - assert!( - info.to_ascii_lowercase() - .contains(&expected_driver.to_ascii_lowercase()), - "gateway info should report the {expected_driver} compute driver: {info}" - ); -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_cannot_inspect_gateway() { - let session = login_identity(USER).await; - let output = run_session_cli(&session, &["gateway", "info"]).await; - let denied = combined_output(&output); - assert!( - !output.status.success(), - "user accessed gateway info:\n{denied}" - ); - assert!( - denied.contains("requires admin privileges"), - "gateway-info denial should explain that admin privileges are required:\n{denied}" - ); - assert_admin_role_denial(&output, "inspect gateway info"); -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_list_providers() { - let session = login_identity(ADMIN).await; - assert_allowed( - &session, - &["provider", "list", "--output", "json"], - "list providers", - ) - .await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_can_list_providers() { - const WORKSPACE: &str = "oidc-user-list-pr"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - assert_workspace_allowed( - &user, - WORKSPACE, - &["provider", "list", "--output", "json"], - "list providers", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_manage_provider() { - const PROVIDER: &str = "oidc-pkce-admin-provider"; - let session = login_identity(ADMIN).await; - - assert_allowed( - &session, - &[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - "create a provider", - ) - .await; - - let get = assert_allowed(&session, &["provider", "get", PROVIDER], "read a provider").await; - assert!( - combined_output(&get).contains(PROVIDER), - "provider get output should contain the created provider:\n{}", - combined_output(&get) - ); - - assert_allowed( - &session, - &["provider", "delete", PROVIDER], - "delete a provider", - ) - .await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_cannot_create_provider() { - const WORKSPACE: &str = "oidc-user-no-create"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - let output = run_workspace_cli( - &user, - WORKSPACE, - &[ - "provider", - "create", - "--name", - "oidc-pkce-user-provider", - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - ) - .await; - assert_workspace_admin_denial(&output, &user, WORKSPACE, "create a provider"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_cannot_delete_provider() { - const PROVIDER: &str = "oidc-pkce-user-delete-target"; - const WORKSPACE: &str = "oidc-user-no-delete"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - assert_workspace_allowed( - &admin, - WORKSPACE, - &[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - "create the provider deletion target", - ) - .await; - - let denied = run_workspace_cli(&user, WORKSPACE, &["provider", "delete", PROVIDER]).await; - assert_workspace_admin_denial(&denied, &user, WORKSPACE, "delete a provider"); - - assert_workspace_allowed( - &admin, - WORKSPACE, - &["provider", "delete", PROVIDER], - "clean up the provider deletion target", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_create_workspace() { - const WORKSPACE: &str = "oidc-admin-create"; - let admin = login_identity(ADMIN).await; - let _ = run_session_cli(&admin, &["workspace", "delete", WORKSPACE]).await; - assert_allowed( - &admin, - &["workspace", "create", "--name", WORKSPACE], - "create a workspace", - ) - .await; - let get = assert_allowed( - &admin, - &["workspace", "get", WORKSPACE], - "read the created workspace", - ) - .await; - assert!(combined_output(&get).contains(WORKSPACE)); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_cannot_create_workspace() { - let user = login_identity(USER).await; - let denied = run_session_cli( - &user, - &["workspace", "create", "--name", "oidc-user-denied"], - ) - .await; - assert_admin_role_denial(&denied, "create a workspace"); -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn admin_can_delete_workspace() { - const WORKSPACE: &str = "oidc-admin-delete"; - let admin = login_identity(ADMIN).await; - let _ = run_session_cli(&admin, &["workspace", "delete", WORKSPACE]).await; - assert_allowed( - &admin, - &["workspace", "create", "--name", WORKSPACE], - "create a workspace deletion target", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn user_cannot_delete_workspace() { - const WORKSPACE: &str = "oidc-user-del-deny"; - let admin = login_identity(ADMIN).await; - let user = login_identity(USER).await; - let _ = run_session_cli(&admin, &["workspace", "delete", WORKSPACE]).await; - assert_allowed( - &admin, - &["workspace", "create", "--name", WORKSPACE], - "create a workspace deletion target", - ) - .await; - let denied = run_session_cli(&user, &["workspace", "delete", WORKSPACE]).await; - assert_admin_role_denial(&denied, "delete a workspace"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_can_read_workspace() { - const WORKSPACE: &str = "oidc-ws-user-read"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - - let get = assert_allowed( - &user, - &["workspace", "get", WORKSPACE], - "read a member workspace", - ) - .await; - let list = assert_allowed( - &user, - &["workspace", "list", "--output", "json"], - "list member workspaces", - ) - .await; - let members = assert_allowed( - &user, - &["workspace", "member", "list", "--workspace", WORKSPACE], - "list workspace members", - ) - .await; - assert!(combined_output(&get).contains(WORKSPACE)); - assert!(combined_output(&list).contains(WORKSPACE)); - assert!(combined_output(&members).contains(&user.subject)); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_cannot_manage_members() { - const WORKSPACE: &str = "oidc-ws-user-deny"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - let denied = run_session_cli( - &user, - &[ - "workspace", - "member", - "add", - "--workspace", - WORKSPACE, - "--subject", - "oidc-fake-member", - "--role", - "user", - ], - ) - .await; - assert_workspace_admin_denial(&denied, &user, WORKSPACE, "add a workspace member"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_read_workspace() { - const WORKSPACE: &str = "oidc-wsa-read"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - - let get = assert_allowed( - &user, - &["workspace", "get", WORKSPACE], - "read an administered workspace", - ) - .await; - assert!(combined_output(&get).contains(WORKSPACE)); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_create_sandbox() { - const WORKSPACE: &str = "oidc-wsa-create-sb"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_create_sandbox(&user, WORKSPACE, "oidc-wsa-create").await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_delete_sandbox() { - const WORKSPACE: &str = "oidc-wsa-delete-sb"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; - assert_can_delete_sandbox(&user, WORKSPACE, "oidc-wsa-delete").await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_create_provider() { - const WORKSPACE: &str = "oidc-wsa-create-pr"; - const PROVIDER: &str = "oidc-wsa-create-provider"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - assert_workspace_allowed( - &user, - WORKSPACE, - &[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - "create a provider as workspace admin", - ) - .await; - - assert_workspace_allowed( - &admin, - WORKSPACE, - &["provider", "delete", PROVIDER], - "clean up the provider created by a workspace admin", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_delete_provider() { - const WORKSPACE: &str = "oidc-wsa-delete-pr"; - const PROVIDER: &str = "oidc-wsa-delete-provider"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - assert_workspace_allowed( - &admin, - WORKSPACE, - &[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - "create the provider deletion target", - ) - .await; - assert_workspace_allowed( - &user, - WORKSPACE, - &["provider", "delete", PROVIDER], - "delete a provider as workspace admin", - ) - .await; - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_add_user_member() { - const WORKSPACE: &str = "oidc-wsa-add-user"; - let workspace_admin = login_identity(USER).await; - let user = login_identity(USER_B).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &workspace_admin, WORKSPACE, "admin").await; - - assert_allowed( - &workspace_admin, - &[ - "workspace", - "member", - "add", - "--workspace", - WORKSPACE, - "--subject", - &user.subject, - "--role", - "user", - ], - "add a standard workspace member", - ) - .await; - let members = assert_allowed( - &workspace_admin, - &["workspace", "member", "list", "--workspace", WORKSPACE], - "list workspace members after adding one", - ) - .await; - assert!(combined_output(&members).contains(&user.subject)); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_can_remove_user_member() { - const WORKSPACE: &str = "oidc-wsa-rm-user"; - let workspace_admin = login_identity(USER).await; - let user = login_identity(USER_B).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &workspace_admin, WORKSPACE, "admin").await; - assert_allowed( - &admin, - &[ - "workspace", - "member", - "add", - "--workspace", - WORKSPACE, - "--subject", - &user.subject, - "--role", - "user", - ], - "create the workspace member removal target", - ) - .await; - - assert_allowed( - &workspace_admin, - &[ - "workspace", - "member", - "remove", - "--workspace", - WORKSPACE, - "--subject", - &user.subject, - ], - "remove a standard workspace member", - ) - .await; - let denied = run_session_cli(&user, &["workspace", "get", WORKSPACE]).await; - assert_non_member_denial(&denied, "read a workspace after removal by its admin"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_grant_admin() { - const WORKSPACE: &str = "oidc-ws-admin-deny"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - let denied = run_session_cli( - &user, - &[ - "workspace", - "member", - "add", - "--workspace", - WORKSPACE, - "--subject", - "oidc-fake-admin", - "--role", - "admin", - ], - ) - .await; - assert_platform_admin_denial(&denied, "grant workspace admin"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_create_workspace() { - const WORKSPACE: &str = "oidc-wsa-no-create"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; - - let denied = - run_session_cli(&user, &["workspace", "create", "--name", "oidc-wsa-denied"]).await; - assert_admin_role_denial(&denied, "create a workspace as workspace admin"); - delete_workspace(&admin, WORKSPACE).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_delete_workspace() { - const WORKSPACE: &str = "oidc-wsa-no-delete"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; +use std::collections::HashMap; +use std::ffi::OsString; +use std::fs::Permissions; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::{Output, Stdio}; +use std::time::{Duration, Instant}; - let denied = run_session_cli(&user, &["workspace", "delete", WORKSPACE]).await; - assert_admin_role_denial(&denied, "delete an administered workspace"); - delete_workspace(&admin, WORKSPACE).await; -} +use base64::Engine as _; +use openshell_e2e::harness::binary::openshell_cmd; +use serde_json::Value; +use serial_test::serial; +use tokio::process::Command; +use tokio::sync::Mutex; +use url::Url; -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_inspect_gateway() { - const WORKSPACE: &str = "oidc-wsa-no-gw-info"; - let user = login_identity(USER).await; - let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "admin").await; +static SANDBOX_LIFECYCLE_LOCK: Mutex<()> = Mutex::const_new(()); - let denied = run_workspace_cli(&user, WORKSPACE, &["gateway", "info"]).await; - assert_admin_role_denial(&denied, "inspect gateway info as workspace admin"); - delete_workspace(&admin, WORKSPACE).await; +#[derive(Clone, Copy)] +struct IdentityScenario { + gateway_name: &'static str, + username: &'static str, + password: &'static str, + expected_role: &'static str, } -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_read_another_workspace() { - const WORKSPACE_A: &str = "oidc-wsa-xread-a"; - const WORKSPACE_B: &str = "oidc-wsa-xread-b"; - let (admin, workspace_admin, _user_b) = - prepare_isolated_workspaces_with_admin(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_session_cli(&workspace_admin, &["workspace", "get", WORKSPACE_B]).await; - assert_non_member_denial(&denied, "read another workspace as workspace admin"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} +const ADMIN: IdentityScenario = IdentityScenario { + gateway_name: "oidc-pkce-admin", + username: "admin@test", + password: "admin", + expected_role: "openshell-admin", +}; -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_admin_cannot_manage_another_workspace_members() { - const WORKSPACE_A: &str = "oidc-wsa-xmem-a"; - const WORKSPACE_B: &str = "oidc-wsa-xmem-b"; - let (admin, workspace_admin, _user_b) = - prepare_isolated_workspaces_with_admin(WORKSPACE_A, WORKSPACE_B).await; +const USER: IdentityScenario = IdentityScenario { + gateway_name: "oidc-pkce-user", + username: "user@test", + password: "user", + expected_role: "openshell-user", +}; - let denied = run_session_cli( - &workspace_admin, - &[ - "workspace", - "member", - "add", - "--workspace", - WORKSPACE_B, - "--subject", - "oidc-fake-member", - "--role", - "user", - ], - ) - .await; - assert_non_member_denial( - &denied, - "manage another workspace's members as workspace admin", - ); +const USER_B: IdentityScenario = IdentityScenario { + gateway_name: "oidc-pkce-user-b", + username: "user-b@test", + password: "user-b", + expected_role: "openshell-user", +}; - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; +struct LoginSession { + config_home: tempfile::TempDir, + identity: IdentityScenario, + subject: String, } #[tokio::test] #[serial(oidc_pkce)] -async fn workspace_admin_cannot_manage_another_workspace_providers() { - const WORKSPACE_A: &str = "oidc-wsa-xprov-a"; - const WORKSPACE_B: &str = "oidc-wsa-xprov-b"; - const PROVIDER: &str = "oidc-wsa-xprovider"; - let (admin, workspace_admin, _user_b) = - prepare_isolated_workspaces_with_admin(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_workspace_cli( - &workspace_admin, - WORKSPACE_B, - &[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=e2e-test-value", - ], - ) - .await; - let diagnostic = combined_output(&denied); - let compact: String = diagnostic - .chars() - .filter(|character| !character.is_whitespace() && *character != '│') - .collect(); - // Profile lookup redacts backend diagnostics. Check the permission code and - // safe recovery guidance without requiring the server's membership details. - assert!( - !denied.status.success() - && compact.contains("PERMISSION_DENIED") - && compact.contains("verifyworkspacemembershipandrequiredpermissions"), - "cross-workspace provider creation did not report a safe permission denial:\n{diagnostic}" - ); - assert!(!diagnostic.contains("e2e-test-value")); - - // Query with an independent authorized identity so a failed command alone - // cannot hide a provider created before the denial was returned. - let listed = assert_workspace_allowed( +async fn admin_can_delete_workspace() { + const WORKSPACE: &str = "oidc-admin-delete"; + let admin = login_identity(ADMIN).await; + let _ = run_session_cli(&admin, &["workspace", "delete", WORKSPACE]).await; + assert_allowed( &admin, - WORKSPACE_B, - &["provider", "list", "--output", "json"], - "verify denied creation left the target workspace empty", + &["workspace", "create", "--name", WORKSPACE], + "create a workspace deletion target", ) .await; - // CLI startup diagnostics may precede the JSON object on stdout. - let stdout = String::from_utf8(listed.stdout).expect("provider list output should be UTF-8"); - let json_start = stdout - .find('{') - .expect("provider list output should contain JSON"); - let json_end = stdout - .rfind('}') - .expect("provider list output should contain a complete JSON object"); - let listing: Value = serde_json::from_str(&stdout[json_start..=json_end]) - .expect("provider list --output json should return JSON on stdout"); - assert_eq!( - listing["next_page_token"], "", - "provider listing is incomplete" - ); - assert!( - listing["providers"] - .as_array() - .expect("provider collection") - .is_empty(), - "denied creation added a provider to the isolated target workspace" - ); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; + delete_workspace(&admin, WORKSPACE).await; } #[tokio::test] #[serial(oidc_pkce)] -async fn membership_removal_revokes_workspace_access() { - const WORKSPACE: &str = "oidc-ws-revoke"; +async fn workspace_admin_can_create_sandbox() { + const WORKSPACE: &str = "oidc-wsa-create-sb"; let user = login_identity(USER).await; let admin = login_identity(ADMIN).await; - prepare_workspace(&admin, &user, WORKSPACE, "user").await; - assert_allowed( - &admin, - &[ - "workspace", - "member", - "remove", - "--workspace", - WORKSPACE, - "--subject", - &user.subject, - ], - "remove a workspace member", - ) - .await; - let denied = run_session_cli(&user, &["workspace", "get", WORKSPACE]).await; - assert_non_member_denial(&denied, "read a workspace after membership removal"); + prepare_workspace(&admin, &user, WORKSPACE, "admin").await; + let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; + assert_can_create_sandbox(&user, WORKSPACE, "oidc-wsa-create").await; delete_workspace(&admin, WORKSPACE).await; } #[tokio::test] #[serial(oidc_pkce)] -async fn workspace_user_cannot_read_another_users_workspace() { - const WORKSPACE_A: &str = "oidc-xread-a"; - const WORKSPACE_B: &str = "oidc-xread-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_session_cli(&user_a, &["workspace", "get", WORKSPACE_B]).await; - assert_non_member_denial(&denied, "read another user's workspace"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; +async fn workspace_admin_can_delete_sandbox() { + const WORKSPACE: &str = "oidc-wsa-delete-sb"; + let user = login_identity(USER).await; + let admin = login_identity(ADMIN).await; + prepare_workspace(&admin, &user, WORKSPACE, "admin").await; + let _lifecycle = SANDBOX_LIFECYCLE_LOCK.lock().await; + assert_can_delete_sandbox(&user, WORKSPACE, "oidc-wsa-delete").await; + delete_workspace(&admin, WORKSPACE).await; } #[tokio::test] @@ -932,117 +120,6 @@ async fn second_workspace_user_cannot_read_first_users_workspace() { delete_workspace(&admin, WORKSPACE_A).await; } -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_list_hides_another_users_workspace() { - const WORKSPACE_A: &str = "oidc-xlist-a"; - const WORKSPACE_B: &str = "oidc-xlist-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let listed = assert_allowed( - &user_a, - &["workspace", "list", "--output", "json"], - "list visible workspaces", - ) - .await; - let output = combined_output(&listed); - assert!( - output.contains(WORKSPACE_A), - "workspace list should contain the caller's workspace:\n{output}" - ); - assert!( - !output.contains(WORKSPACE_B), - "workspace list exposed another user's workspace:\n{output}" - ); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_cannot_list_another_workspace_sandboxes() { - const WORKSPACE_A: &str = "oidc-xsbox-a"; - const WORKSPACE_B: &str = "oidc-xsbox-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_workspace_cli( - &user_a, - WORKSPACE_B, - &["sandbox", "list", "--output", "json"], - ) - .await; - assert_non_member_denial(&denied, "list another workspace's sandboxes"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_cannot_create_sandbox_in_another_workspace() { - const WORKSPACE_A: &str = "oidc-xcreate-a"; - const WORKSPACE_B: &str = "oidc-xcreate-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_workspace_cli( - &user_a, - WORKSPACE_B, - &[ - "sandbox", - "create", - "--name", - "oidc-xcreate-denied", - "--no-tty", - "--", - "echo", - "denied", - ], - ) - .await; - assert_non_member_denial(&denied, "create a sandbox in another workspace"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_cannot_list_another_workspace_providers() { - const WORKSPACE_A: &str = "oidc-xprov-a"; - const WORKSPACE_B: &str = "oidc-xprov-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_workspace_cli( - &user_a, - WORKSPACE_B, - &["provider", "list", "--output", "json"], - ) - .await; - assert_non_member_denial(&denied, "list another workspace's providers"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} - -#[tokio::test] -#[serial(oidc_pkce)] -async fn workspace_user_cannot_list_another_workspace_members() { - const WORKSPACE_A: &str = "oidc-xmember-a"; - const WORKSPACE_B: &str = "oidc-xmember-b"; - let (admin, user_a, _user_b) = prepare_isolated_workspaces(WORKSPACE_A, WORKSPACE_B).await; - - let denied = run_session_cli( - &user_a, - &["workspace", "member", "list", "--workspace", WORKSPACE_B], - ) - .await; - assert_non_member_denial(&denied, "list another workspace's members"); - - delete_workspace(&admin, WORKSPACE_B).await; - delete_workspace(&admin, WORKSPACE_A).await; -} - async fn login_identity(identity: IdentityScenario) -> LoginSession { let issuer = std::env::var("OPENSHELL_E2E_OIDC_ISSUER") .unwrap_or_else(|_| "http://localhost:8180/realms/openshell".to_string()); @@ -1365,22 +442,6 @@ async fn assert_allowed(session: &LoginSession, args: &[&str], action: &str) -> output } -async fn assert_workspace_allowed( - session: &LoginSession, - workspace: &str, - args: &[&str], - action: &str, -) -> Output { - let output = run_workspace_cli(session, workspace, args).await; - assert!( - output.status.success(), - "{} should be allowed to {action} in workspace {workspace}:\n{}", - session.identity.username, - combined_output(&output) - ); - output -} - async fn prepare_workspace( admin: &LoginSession, member: &LoginSession, @@ -1424,18 +485,6 @@ async fn prepare_isolated_workspaces( (admin, user_a, user_b) } -async fn prepare_isolated_workspaces_with_admin( - workspace_a: &str, - workspace_b: &str, -) -> (LoginSession, LoginSession, LoginSession) { - let admin = login_identity(ADMIN).await; - let workspace_admin = login_identity(USER).await; - let user_b = login_identity(USER_B).await; - prepare_workspace(&admin, &workspace_admin, workspace_a, "admin").await; - prepare_workspace(&admin, &user_b, workspace_b, "user").await; - (admin, workspace_admin, user_b) -} - async fn delete_workspace(admin: &LoginSession, workspace: &str) { for attempt in 0..30 { let output = run_session_cli(admin, &["workspace", "delete", workspace]).await; @@ -1590,55 +639,6 @@ async fn run_workspace_cli(session: &LoginSession, workspace: &str, args: &[&str run_cli(session.config_home.path(), &command_args).await } -fn assert_admin_role_denial(output: &Output, action: &str) { - let denied = combined_output(output); - let compact_denial: String = denied - .chars() - .filter(|character| !character.is_whitespace() && *character != '│') - .collect(); - assert!( - !output.status.success() && compact_denial.contains("openshell-admin"), - "standard user unexpectedly authorized to {action}, or denial omitted the admin role:\n{denied}" - ); -} - -fn assert_workspace_admin_denial( - output: &Output, - session: &LoginSession, - workspace: &str, - action: &str, -) { - let denied = combined_output(output); - let remediation = format!( - "openshell workspace member add --workspace '{workspace}' --subject '{}' --role admin", - session.subject - ); - let compact_denial: String = denied - .chars() - .filter(|character| !character.is_whitespace() && *character != '│') - .collect(); - let compact_remediation: String = remediation - .chars() - .filter(|character| !character.is_whitespace()) - .collect(); - assert!( - !output.status.success() - && compact_denial - .to_ascii_lowercase() - .contains("workspacerole'admin'") - && compact_denial.contains(&compact_remediation), - "workspace user unexpectedly authorized to {action}, or denial omitted the admin remediation command:\n{denied}" - ); -} - -fn assert_platform_admin_denial(output: &Output, action: &str) { - let denied = combined_output(output); - assert!( - !output.status.success() && denied.to_ascii_lowercase().contains("platform admin"), - "non-platform-admin unexpectedly authorized to {action}, or denial omitted the required platform role:\n{denied}" - ); -} - fn assert_non_member_denial(output: &Output, action: &str) { let denied = combined_output(output); let compact_denial: String = denied From c29be496224bd809f091d66ea262a8ee85391950 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 17:08:20 -0800 Subject: [PATCH 48/63] docs(testing): describe the OIDC feature environment Signed-off-by: Matthew Grossman --- TESTING.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/TESTING.md b/TESTING.md index 33b203073b..9fe63b8542 100644 --- a/TESTING.md +++ b/TESTING.md @@ -337,6 +337,15 @@ nix run .#openshell-test-runner -- run \ --family features --package keycloak-provider-refresh ``` +The `oidc-auth` suite runs in `ubuntu-docker-rootful-oidc`, which adds a gateway +that trusts the Keycloak realm and gives testkit an operator client certificate: + +```shell +nix run .#openshell-test-runner -- run \ + --environment ubuntu-docker-rootful-oidc --installer deb \ + --family features --package oidc-auth +``` + A suite that needs setup beyond a container engine runs in a feature environment, named `-`, such as `ubuntu-docker-rootful-keycloak` or `ubuntu-docker-rootful-cdi`. `features` in `tests/tmachine/config.nix` lists From c67e467b49ee9d0e9be83d67a3f66114b17d1fff Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:13:47 -0800 Subject: [PATCH 49/63] test(features): use the host's outbound address when routable-host-ip arrives on loopback Signed-off-by: Matthew Grossman --- TESTING.md | 10 +++--- .../tests/routable_host_ip.rs | 32 +++++++++++++++---- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/TESTING.md b/TESTING.md index e234086747..8bb56da91e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -658,11 +658,11 @@ the node's address. The `routable-host-ip` feature suite (`tests/suites/features/routable-host-ip`) checks that a policy endpoint whose host is an IP literal reaches that address, with or without `allowed_ips`. The test reaches a host server by -`host.openshell.internal`, then uses the address that request arrived at, which -must be one the supervisor lets an IP literal reach. Rootless Podman's is -link-local and always blocked, and Docker Desktop's is loopback, so CI runs the -suite on Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` -installers. +`host.openshell.internal`, then uses the address that request arrived at, or +the host's outbound address when it arrived on loopback from a host-network +supervisor. The address must be one the supervisor lets an IP literal reach. +Rootless Podman's is link-local and always blocked, so CI runs the suite on +Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` installers. The `connect-ssh-supervision` feature suite (`tests/suites/features/connect-ssh-supervision`) checks how `sandbox connect` diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs index b05e07a664..68e0996e42 100644 --- a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -7,13 +7,18 @@ //! The servers run on the test host. The sandbox can't learn the host's //! address, since policy DNS answers it with a synthetic one, so the test //! reaches a server by `host.openshell.internal` first and takes the address -//! the server saw the request arrive at. Not conformance, because +//! the server saw the request arrive at. A supervisor on the host network +//! (Docker's and Podman's default) arrives on loopback, which an IP literal +//! can't reach, so the test uses the host's outbound address instead; that +//! supervisor shares it. Not conformance, because //! that address must be one the supervisor lets an IP literal reach: rootless //! Podman's is link-local, which is always blocked. //! //! Ported from the `proxy_egress_pipeline` e2e test of IP literals, which //! used two support containers on the e2e network instead. +use std::net::{IpAddr, Ipv4Addr, UdpSocket}; + use host_server_support::{HttpRecorder, TEST_HOST}; use sandbox_clients_support::{Request, http}; use testkit::policy; @@ -52,17 +57,17 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul "request to {TEST_HOST}:{} failed: {reply:?}", explicit.port ); - let host_ip = explicit + let arrived_on = explicit .requests() .first() .and_then(|request| request.local_addr) .ok_or_else(|| testkit::Error::msg("the host server didn't record its address"))? .ip(); - // Docker Desktop relays the host gateway through a host-side proxy. - testkit::ensure!( - !host_ip.is_loopback(), - "requests to {TEST_HOST} arrive at the host on {host_ip}, which a sandbox can't reach" - ); + let host_ip = if arrived_on.is_loopback() { + outbound_address()? + } else { + arrived_on + }; let ip = host_ip.to_string(); let prefix = if host_ip.is_ipv4() { 32 } else { 128 }; @@ -96,3 +101,16 @@ async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Resul } Ok(()) } + +/// The test host's address on its default route. Connecting a UDP socket +/// sends nothing; it only picks the route. +fn outbound_address() -> testkit::Result { + let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0))?; + socket.connect((Ipv4Addr::new(192, 0, 2, 1), 9))?; + let address = socket.local_addr()?.ip(); + testkit::ensure!( + !address.is_loopback() && !address.is_unspecified(), + "the test host has no outbound address, only {address}" + ); + Ok(address) +} From f4689360f749fe39859e0b15e167ce888b3530af Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:14:30 -0800 Subject: [PATCH 50/63] ci(e2e): run service-exposure only where gateway service routing is on Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 2 -- TESTING.md | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index a9a6f53b1c..0530b9a55f 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -108,8 +108,6 @@ jobs: {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, - {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, - {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 3ec9d9ddd3..36b9f4b4ea 100644 --- a/TESTING.md +++ b/TESTING.md @@ -665,7 +665,9 @@ The `service-exposure` feature suite (`tests/suites/features/service-exposure`) exposes a service in a sandbox and checks how the gateway forwards the client's `Authorization` header. The test sends requests to the gateway's endpoint with the service URL's host as the Host header, so the test machine doesn't resolve -service hosts. CI runs it on the same environments as `host-gateway`. +service hosts. CI runs it on Docker rootful and Podman rootful and rootless. The +Helm chart turns gateway service routing off, so the K3s installers have no +service URLs. Run the VM-backed Rust CLI e2e suite: From 89975a46c6b566c8933885162828bb80bf4a3b27 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 51/63] docs(testkit): add the tests/support environment contract table Signed-off-by: Matthew Grossman --- tests/support/README.md | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/support/README.md diff --git a/tests/support/README.md b/tests/support/README.md new file mode 100644 index 0000000000..f2ca63d396 --- /dev/null +++ b/tests/support/README.md @@ -0,0 +1,42 @@ +# Test support crates + +Each crate here is a fixture for one concern, such as a service the +environment runs, and any suite can depend on it by path. See +[Adding a feature suite](../../TESTING.md#adding-a-feature-suite) for how an +environment and a fixture share a contract. + +## Environment contract + +These are the variables that an environment, the test runner or CI sets for +tests to read. A tmachine environment publishes its variables with +`tests/tmachine/ansible/playbooks/test-env.yaml`, and `openshell-test-runner` +passes them to the tests. To run a suite against your own deployment, set them +yourself. A fixture fails the test when a variable it needs is missing; it +never skips it. Add a row when you add a variable. + +| Variable | Set by | Read by | Meaning | +|---|---|---|---| +| `OPENSHELL_TEST_CONTAINER_ENGINE` | Every Docker and Podman base environment (`playbooks/docker.yaml`, `podman-rootful.yaml`, `podman-rootless.yaml`); `e2e:oci-image` tasks | `container-engine-support` | The command that reaches the gateway's container engine, split on whitespace, such as `docker` or `sudo -n podman`. | +| `OPENSHELL_TEST_PODMAN_USERNS` | `-userns-` (`playbooks/drivers/podman/userns.yaml`) | `podman-support` | The gateway's Podman user-namespace mode: `default`, `auto`, `keep-id` or `private`. | +| `OPENSHELL_TEST_CDI_SCENARIOS` | `-cdi` (`playbooks/cdi.yaml`) | `tests/suites/features/cdi` | The path of the document that names the CDI device kind, each scenario and the checks it expects. | +| `OPENSHELL_TEST_KEYCLOAK_ISSUER` | `-keycloak` (`playbooks/keycloak.yaml`) | `keycloak-support` | The OIDC issuer URL of the test realm, reachable from the test process. | +| `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | +| `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | +| `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | +| `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | +| `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | +| `OPENSHELL_TEST_POSTGRES_URL` | `tasks/scripts/run-postgres-tests.sh` (`mise run test:rust:postgres`), or you | `openshell-server`'s PostgreSQL tests | A disposable PostgreSQL database. Never point it at one a gateway uses. | + +Other `OPENSHELL_TEST_*` names are internal test hooks and are left out on +purpose: unit-test child-process markers, the re-entry markers in +`tests/support/environment.rs`, Nix guest-cache settings, `install.sh` test +hooks and crate test settings. + +### Addresses the environment chooses + +When a test must listen on an address the environment picked, because the +gateway or another service is configured to call it, the environment +publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it. The +test never picks the address itself. Tests that bind the same address can't +run at the same time. From d90a51955c7d97f27265e51cb2bb192f6c25d088 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 52/63] test(tmachine): add a gateway_overrides role for environment gateway settings Signed-off-by: Matthew Grossman --- TESTING.md | 29 ++++++++--- .../playbooks/drivers/podman/userns.yaml | 42 ++++----------- .../roles/cdi_fixtures/tasks/main.yaml | 24 ++------- .../gateway_overrides/defaults/main.yaml | 5 ++ .../roles/gateway_overrides/tasks/main.yaml | 51 +++++++++++++++++++ .../openshell_external_driver/tasks/main.yaml | 17 ++----- .../files/merge_gateway_toml_test.py | 22 ++++++++ .../tasks/merge-settings.yaml | 7 +-- 8 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml diff --git a/TESTING.md b/TESTING.md index 8bb56da91e..24e5fe6f05 100644 --- a/TESTING.md +++ b/TESTING.md @@ -324,7 +324,9 @@ service. clients there from the variables. A missing variable fails the test; it never skips it. - **Documentation.** Document the variables in the fixture's module docs, so - anyone can run the suite against their own deployment by setting them. + anyone can run the suite against their own deployment by setting them, and + add a row for each to the + [environment contract table](tests/support/README.md#environment-contract). The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it with: @@ -596,11 +598,26 @@ nix run .#openshell-test-runner -- run \ #### Gateway overrides in an environment -An environment overrides gateway settings by writing the tables to change to -`/etc/openshell-test/gateway-overrides.toml` in a playbook that runs before the -installer. The file holds whole tables, such as `[openshell.gateway]` or -`[openshell.drivers.podman]`, and is not a complete configuration. Each -installer applies it to the gateway's own file: +An environment overrides gateway settings with the `gateway_overrides` role, +which takes a name and the tables to change, such as `[openshell.gateway]` or +`[openshell.drivers.podman]`: + +```yaml +- name: Push configuration to supervisors + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: config-push + gateway_overrides_content: | + [openshell.gateway] + config_delivery_mode = "push" +``` + +The role writes each name as its own marked block in +`/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's +settings. Two blocks must not declare the same table. Don't write the file +directly. The file is not a complete configuration. Each installer applies it +to the gateway's own file: - `binaries` applies it to the `/etc/openshell/gateway.toml` it generates. - `deb` and `rpm` apply it to the service user's diff --git a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml index 202270100f..dc74fe8a93 100644 --- a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml +++ b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml @@ -4,18 +4,16 @@ # Configure the Podman gateway's user-namespace mode and publish it. # # `podman_userns_mode` is default, auto, keep-id or private. Every mode except -# default writes its `[openshell.drivers.podman]` settings to -# `/etc/openshell-test/gateway-overrides.toml`, which every installer applies to the active -# gateway-overrides.toml (the openshell_gateway role for `binaries`, the -# openshell_packaged_gateway role for the packages); default removes the file. The -# Podman support crate (tests/support/podman) reads the published mode and -# runs Podman's own reference container in it. +# default adds its `[openshell.drivers.podman]` settings to the gateway +# overrides (the gateway_overrides role), which every installer applies to the +# gateway's configuration. Default mode adds nothing, so userns, uidmap and +# gidmap stay unset. The Podman support crate (tests/support/podman) reads the +# published mode and runs Podman's own reference container in it. --- - name: Podman user namespaces for tests hosts: all gather_facts: false vars: - podman_userns_driver_config: /etc/openshell-test/gateway-overrides.toml podman_userns_settings: default: "" auto: | @@ -43,34 +41,14 @@ {{ podman_userns_settings.keys() | join(', ') }}, not {{ podman_userns_mode | default('unset') }} - - name: Create the test configuration directory - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config | dirname }}" - state: directory - owner: root - group: root - mode: "0755" - - name: Write the Podman driver user-namespace settings - become: true - ansible.builtin.copy: - dest: "{{ podman_userns_driver_config }}" - content: "{{ podman_userns_settings[podman_userns_mode] }}" - owner: root - group: root - mode: "0644" + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: podman-userns + gateway_overrides_content: "{{ podman_userns_settings[podman_userns_mode] }}" when: podman_userns_mode != "default" - # The fragment is the only source of userns, uidmap and gidmap settings in - # the gateway's configuration, so default mode leaves them unset. - - name: Remove Podman driver user-namespace settings for default mode - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config }}" - state: absent - when: podman_userns_mode == "default" - - name: Publish Podman user-namespace test variables ansible.builtin.include_tasks: ../../test-env.yaml vars: diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml index 5f0c04586c..250629d90b 100644 --- a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml @@ -38,25 +38,11 @@ # The tests select the scenario device per sandbox through driver config, which # the gateway rejects unless an administrator allows it. The installers add # this to the active driver's settings. -- name: Create the test settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - owner: root - group: root - mode: "0755" - -# The gateway must accept a per-sandbox CDI device, which it does only when the -# driver allows caller driver config. Every installer merges this override into -# the gateway's own configuration (see openshell_gateway/tasks/merge-settings.yaml). - name: Allow caller driver config for the CDI tests - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: cdi + gateway_overrides_content: | [openshell.drivers.{{ environment_driver }}] allow_driver_config = true - owner: root - group: root - mode: "0644" diff --git a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml new file mode 100644 index 0000000000..73e224fb07 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +gateway_overrides_path: /etc/openshell-test/gateway-overrides.toml diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml new file mode 100644 index 0000000000..5114e90e69 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Add gateway settings to /etc/openshell-test/gateway-overrides.toml, which +# the `binaries`, `deb` and `rpm` installers apply to the gateway's +# configuration (openshell_gateway/tasks/merge-settings.yaml; k3s installers +# ignore it). +# +# Inputs: +# gateway_overrides_name names the marked block, such as `cdi`; writing +# the same name again replaces its block +# gateway_overrides_content whole TOML tables, such as +# `[openshell.drivers.podman]` +# +# Each name gets its own marked block, so writers keep each other's settings +# and may run in any order. Two blocks must not declare the same table: TOML +# rejects a table defined twice, and the installer fails on it. +# +# TODO: https://github.com/NVIDIA/OpenShell/pull/5489 reads gateway.d drop-in +# files. When it lands, write each name to its own drop-in here instead. +--- +- name: Require a gateway override name and content + ansible.builtin.assert: + that: + - gateway_overrides_name is defined + - gateway_overrides_name is match('^[a-z0-9][a-z0-9-]*$') + - gateway_overrides_content is defined + - gateway_overrides_content | trim | length > 0 + fail_msg: >- + gateway_overrides needs gateway_overrides_name (lowercase letters, + digits and dashes) and non-empty gateway_overrides_content + +- name: Create the test configuration directory + become: true + ansible.builtin.file: + path: "{{ gateway_overrides_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Write the {{ gateway_overrides_name }} gateway overrides + become: true + ansible.builtin.blockinfile: + path: "{{ gateway_overrides_path }}" + create: true + owner: root + group: root + mode: "0644" + marker: "# {mark} {{ gateway_overrides_name }}" + block: "{{ gateway_overrides_content }}" diff --git a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml index a31363d57d..2f3d4e3872 100644 --- a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml @@ -108,18 +108,11 @@ ansible.builtin.fail: msg: The external driver socket did not appear -- name: Create the environment settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - mode: "0755" - - name: Point the gateway at the external driver - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - mode: "0644" - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: external-driver + gateway_overrides_content: | [openshell.drivers.{{ openshell_external_driver_name }}] socket_path = "{{ openshell_external_driver_socket }}" diff --git a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py index ba6926169a..3f37c15178 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py +++ b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py @@ -98,3 +98,25 @@ def test_creates_from_default_else_minimal(tmp_path): run, base = merge(other, None, '[openshell.drivers.podman]\nuserns = "auto"\n') parsed = tomllib.loads(base.read_text()) assert parsed["openshell"]["version"] == 2 + + +def test_applies_every_marked_block(tmp_path): + overrides = ( + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + "# BEGIN config-push\n[openshell.gateway]\nconfig_delivery_mode = \"push\"\n# END config-push\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode == 0, run.stderr + parsed = tomllib.loads(base.read_text()) + assert parsed["openshell"]["drivers"]["podman"]["userns"] == "auto" + assert parsed["openshell"]["gateway"]["config_delivery_mode"] == "push" + + +def test_rejects_blocks_that_declare_the_same_table(tmp_path): + overrides = ( + "# BEGIN cdi\n[openshell.drivers.podman]\nallow_driver_config = true\n# END cdi\n" + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode != 0 + assert base.read_text() == BASE diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml index 6d167a3ec8..2b3b8e3e22 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml @@ -1,10 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# An environment overrides gateway settings by writing the tables to change, +# An environment overrides gateway settings by adding the tables to change, # such as `[openshell.drivers.podman]`, to -# /etc/openshell-test/gateway-overrides.toml before the installer runs (for -# example playbooks/drivers/podman/userns.yaml). Apply them to +# /etc/openshell-test/gateway-overrides.toml with the gateway_overrides role +# before the installer runs (for example playbooks/drivers/podman/userns.yaml). +# Apply them to # `openshell_gateway_merge_target` without replacing the file; see # files/merge-gateway-toml.py. # From a7e7bb90be75d6e9385400301ee4e76cafb4c677 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:00:13 -0800 Subject: [PATCH 53/63] test(testkit): serve TLS from host servers with a test or environment CA Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 4 + tests/suites/features/Cargo.lock | 4 + tests/support/Cargo.lock | 4 + tests/support/README.md | 2 + tests/support/host-server/Cargo.toml | 4 + tests/support/host-server/src/http.rs | 72 ++++- tests/support/host-server/src/lib.rs | 5 +- tests/support/host-server/src/tls.rs | 308 +++++++++++++++++++++ tests/support/host-server/src/websocket.rs | 3 +- 9 files changed, 391 insertions(+), 15 deletions(-) create mode 100644 tests/support/host-server/src/tls.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 0d17b24979..53c4bdee9e 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1285,9 +1285,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 3a0b5a30b2..2f966d222c 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1317,9 +1317,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 044a44aa3e..440b9ec84b 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1281,9 +1281,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/README.md b/tests/support/README.md index f2ca63d396..c13ba758e1 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -23,6 +23,8 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | +| `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | | `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | | `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index cf51fd6a4c..ccac94f0d1 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -12,9 +12,13 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } +rcgen = { version = "0.14.10", default-features = false, features = ["pem"] } +rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } serde_json = "1" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["tls12"] } tungstenite = "0.26" [dev-dependencies] diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index b93e6dddcc..8b1f830faa 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -7,10 +7,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::TcpStream; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; use tokio::task::JoinHandle; +use tokio_rustls::TlsAcceptor; +use crate::tls::Identity; use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; /// A request an [`HttpRecorder`] read. A chunked body arrives decoded. @@ -171,6 +172,30 @@ impl HttpRecorder { /// Answer every request with `handler`'s response. pub async fn serve( handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(None, handler).await + } + + /// Answer every request over TLS with `200` and `body`, presenting + /// `identity`'s certificate. + pub async fn start_tls(identity: &Identity, body: impl Into) -> Result { + let body = body.into(); + Self::serve_tls(identity, move |_| Response::text(body.clone())).await + } + + /// Answer every request over TLS with `handler`'s response, presenting + /// `identity`'s certificate. A connection that fails the TLS handshake + /// counts in [`connections`](Self::connections) but records no request. + pub async fn serve_tls( + identity: &Identity, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(Some(identity.acceptor()?), handler).await + } + + async fn listen( + tls: Option, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { let (listener, port) = bind().await?; let handler: Arc = Arc::new(handler); @@ -181,11 +206,21 @@ impl HttpRecorder { let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { task_connections.fetch_add(1, Ordering::AcqRel); - tokio::spawn(respond( - stream, - Arc::clone(&handler), - Arc::clone(&task_requests), - )); + let _ = stream.set_nodelay(true); + let local_addr = stream.local_addr().ok(); + let handler = Arc::clone(&handler); + let requests = Arc::clone(&task_requests); + let tls = tls.clone(); + tokio::spawn(async move { + match tls { + None => respond(stream, local_addr, handler, requests).await, + Some(tls) => { + if let Ok(stream) = tls.accept(stream).await { + respond(stream, local_addr, handler, requests).await; + } + } + } + }); } }); Ok(Self { @@ -239,10 +274,14 @@ impl Drop for HttpRecorder { /// Read requests, record each, and answer with `handler` until a response /// or request closes the connection. -async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc>>) { - let _ = stream.set_nodelay(true); +async fn respond( + mut stream: impl AsyncRead + AsyncWrite + Unpin, + local_addr: Option, + handler: Arc, + requests: Arc>>, +) { let mut buffer = Vec::new(); - while let Some(request) = read_request(&mut stream, &mut buffer).await { + while let Some(request) = read_request(&mut stream, local_addr, &mut buffer).await { let response = handler(&request); let close = !response.keep_alive || request @@ -260,13 +299,18 @@ async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc) -> Option { +/// `buffer`, and leave any bytes after it in `buffer`. `local_addr` is the +/// address the connection arrived on. +pub(crate) async fn read_request( + stream: &mut (impl AsyncRead + Unpin), + local_addr: Option, + buffer: &mut Vec, +) -> Option { let mut chunk = [0_u8; 4096]; loop { if let Some((mut request, length)) = parse_request(buffer) { buffer.drain(..length); - request.local_addr = stream.local_addr().ok(); + request.local_addr = local_addr; return Some(request); } match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { @@ -337,6 +381,8 @@ fn decode_chunked(bytes: &[u8]) -> Option<(Vec, usize)> { #[cfg(test)] mod tests { + use tokio::net::TcpStream; + use super::*; async fn connect(port: u16) -> TcpStream { diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index e9781bf502..374d3be419 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -11,7 +11,8 @@ //! host, such as one on a remote cluster, can't run tests that use them. //! //! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's -//! [`Response`], and records each parsed [`Request`]. +//! [`Response`], and records each parsed [`Request`]. It serves HTTPS with +//! a certificate from a [`TestCa`]. //! - [`WebSocketRecorder`] completes the WebSocket handshake and records the //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the @@ -21,6 +22,7 @@ mod http; mod tcp; +mod tls; mod websocket; use std::time::Duration; @@ -30,6 +32,7 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request, Response}; pub use tcp::{TcpServer, exchange}; +pub use tls::{ADDITIONAL_CA_CERT_ENV, ADDITIONAL_CA_KEY_ENV, Identity, TestCa}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; /// The name a sandbox reaches these servers by. diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs new file mode 100644 index 0000000000..6b36293b92 --- /dev/null +++ b/tests/support/host-server/src/tls.rs @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Certificate authorities for TLS host servers. +//! +//! A test signs a server certificate with a [`TestCa`] and serves it with +//! [`HttpRecorder::serve_tls`](crate::HttpRecorder::serve_tls): +//! +//! - [`TestCa::new`] makes a CA that only the test knows, so nothing trusts it +//! unless the test hands over [`TestCa::cert_pem`]. +//! - [`TestCa::from_env`] loads the environment's additional CA, which the +//! gateway is configured to trust (`additional_ca_bundle`). The environment +//! publishes the paths of its PEM certificate and key in +//! `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY`. + +use std::path::Path; +use std::sync::Arc; + +use openshell_crypto::pki::{self, KeyPair}; +use rcgen::{ + BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, + KeyUsagePurpose, +}; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use testkit::{Error, Result}; +use tokio_rustls::TlsAcceptor; + +/// The path of the environment's additional CA certificate, in PEM. +pub const ADDITIONAL_CA_CERT_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_CERT"; +/// The path of the environment's additional CA private key, in PEM. +pub const ADDITIONAL_CA_KEY_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_KEY"; + +/// A certificate authority that signs server certificates for host servers. +pub struct TestCa { + cert_pem: String, + issuer: Issuer<'static, KeyPair>, +} + +impl TestCa { + /// A new CA that only this test knows. + pub fn new() -> Result { + let key = pki::generate_keypair().map_err(failed("generate a test CA key"))?; + let cert = pki::self_signed(ca_params(), &key).map_err(failed("sign the test CA"))?; + Ok(Self { + cert_pem: cert.pem(), + issuer: cert.into_issuer(key), + }) + } + + /// The environment's additional CA, from the files that + /// `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY` + /// name. Fails if either is unset. + pub fn from_env() -> Result { + Self::from_vars(|name| std::env::var(name).ok()) + } + + fn from_vars(var: impl Fn(&str) -> Option) -> Result { + let path = |name: &str| { + var(name) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + Error::msg(format!( + "{name} is not set; run this test in an environment with an additional CA" + )) + }) + }; + let cert = path(ADDITIONAL_CA_CERT_ENV)?; + let key = path(ADDITIONAL_CA_KEY_ENV)?; + Self::from_files(cert.as_ref(), key.as_ref()) + } + + /// A CA from its PEM certificate and private key files. + pub fn from_files(cert: &Path, key: &Path) -> Result { + let read = |path: &Path| { + std::fs::read_to_string(path) + .map_err(|error| Error::msg(format!("read {}: {error}", path.display()))) + }; + let cert_pem = read(cert)?; + let key_pem = read(key)?; + let der = CertificateDer::from_pem_slice(cert_pem.as_bytes()).map_err(failed(format!( + "parse the CA certificate {}", + cert.display() + )))?; + let key = KeyPair::from_pem(&key_pem) + .map_err(failed(format!("parse the CA key {}", key.display())))?; + let issuer = pki::issuer_from_der(&der, key) + .map_err(failed(format!("load the CA {}", cert.display())))?; + Ok(Self { cert_pem, issuer }) + } + + /// The CA certificate, in PEM, for a client that should trust it. + pub fn cert_pem(&self) -> &str { + &self.cert_pem + } + + /// A server certificate for `names`, signed by this CA. A name that parses + /// as an IP address becomes an IP address entry. + pub fn leaf(&self, names: &[&str]) -> Result { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let first = names + .first() + .cloned() + .ok_or_else(|| Error::msg("a server certificate needs at least one name"))?; + let mut params = CertificateParams::new(names.clone()) + .map_err(failed(format!("server certificate names {names:?}")))?; + params.distinguished_name.push(DnType::CommonName, first); + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let key = pki::generate_keypair().map_err(failed("generate a server key"))?; + let cert = pki::signed_by_issuer(params, &key, &self.issuer) + .map_err(failed(format!("sign a server certificate for {names:?}")))?; + let key_der = key + .serialize_der() + .map_err(failed("export the server key"))?; + let key_pem = key + .serialize_pem() + .map_err(failed("export the server key"))?; + Ok(Identity { + cert_pem: cert.pem(), + key_pem, + cert_der: cert.der().clone(), + key_der, + }) + } +} + +/// A server certificate and its private key. +#[derive(Clone)] +pub struct Identity { + pub cert_pem: String, + pub key_pem: String, + cert_der: CertificateDer<'static>, + key_der: Vec, +} + +impl Identity { + pub(crate) fn acceptor(&self) -> Result { + let key = PrivateKeyDer::Pkcs8(self.key_der.clone().into()); + let mut config = openshell_crypto::tls::try_server_builder() + .map_err(failed("build a TLS server configuration"))? + .with_no_client_auth() + .with_single_cert(vec![self.cert_der.clone()], key) + .map_err(failed("load the server certificate"))?; + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(config))) + } +} + +impl std::fmt::Debug for Identity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Identity") + .field("cert_pem", &self.cert_pem) + .finish_non_exhaustive() + } +} + +fn ca_params() -> CertificateParams { + let mut params = CertificateParams::default(); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell test CA"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + KeyUsagePurpose::DigitalSignature, + ]; + params +} + +fn failed(context: impl std::fmt::Display) -> impl FnOnce(E) -> Error { + move |error| Error::msg(format!("{context}: {error}")) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rustls::pki_types::ServerName; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + use tokio_rustls::client::TlsStream; + + use super::*; + use crate::{HttpRecorder, Response}; + + /// Open a TLS connection to `port` on loopback as `name`, trusting `ca`. + async fn connect(ca: &str, name: &str, port: u16) -> std::io::Result> { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.as_bytes()).unwrap()) + .unwrap(); + let config = openshell_crypto::tls::try_client_builder() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await?; + stream.set_nodelay(true)?; + TlsConnector::from(Arc::new(config)) + .connect(ServerName::try_from(name.to_string()).unwrap(), stream) + .await + } + + async fn get(stream: &mut TlsStream, path: &str) -> String { + stream + .write_all(format!("GET {path} HTTP/1.1\r\nHost: test\r\n\r\n").as_bytes()) + .await + .unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + String::from_utf8_lossy(&response).into_owned() + } + + fn write_ca(dir: &str) -> (PathBuf, PathBuf, String) { + let key = pki::generate_keypair().unwrap(); + let cert = pki::self_signed(ca_params(), &key).unwrap(); + let dir = std::env::temp_dir().join(format!("{dir}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (cert_path, key_path) = (dir.join("ca.pem"), dir.join("ca-key.pem")); + std::fs::write(&cert_path, cert.pem()).unwrap(); + std::fs::write(&key_path, key.serialize_pem().unwrap()).unwrap(); + (cert_path, key_path, cert.pem()) + } + + #[tokio::test] + async fn serve_tls_answers_a_client_that_trusts_the_ca() { + let ca = TestCa::new().unwrap(); + let identity = ca.leaf(&["localhost", "127.0.0.1"]).unwrap(); + let recorder = HttpRecorder::serve_tls(&identity, |request| { + Response::text(format!("tls {}", request.path())) + }) + .await + .unwrap(); + for name in ["localhost", "127.0.0.1"] { + let mut stream = connect(ca.cert_pem(), name, recorder.port).await.unwrap(); + let response = get(&mut stream, "/hello").await; + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.ends_with("tls /hello"), "{response}"); + } + let requests = recorder.wait_for_requests(2).await.unwrap(); + assert_eq!(requests[0].path(), "/hello"); + } + + #[tokio::test] + async fn serve_tls_is_rejected_for_another_name_or_ca() { + let ca = TestCa::new().unwrap(); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "ok") + .await + .unwrap(); + let wrong_name = connect(ca.cert_pem(), "other.test", recorder.port).await; + assert!( + wrong_name.is_err(), + "a certificate for another name was accepted" + ); + let other = TestCa::new().unwrap(); + let wrong_ca = connect(other.cert_pem(), "localhost", recorder.port).await; + assert!( + wrong_ca.is_err(), + "a certificate from another CA was accepted" + ); + assert!(recorder.requests().is_empty()); + } + + #[tokio::test] + async fn from_vars_loads_the_environment_ca() { + let (cert, key, cert_pem) = write_ca("host-server-env-ca"); + let ca = TestCa::from_vars(|name| match name { + ADDITIONAL_CA_CERT_ENV => Some(cert.display().to_string()), + ADDITIONAL_CA_KEY_ENV => Some(key.display().to_string()), + _ => None, + }) + .unwrap(); + assert_eq!(ca.cert_pem(), cert_pem); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "env") + .await + .unwrap(); + let mut stream = connect(&cert_pem, "localhost", recorder.port) + .await + .unwrap(); + assert!(get(&mut stream, "/").await.ends_with("env")); + } + + #[test] + fn from_vars_fails_when_a_variable_is_missing() { + let error = TestCa::from_vars(|name| { + (name == ADDITIONAL_CA_CERT_ENV).then(|| "/nonexistent/ca.pem".to_string()) + }) + .err() + .unwrap(); + assert!(error.to_string().contains(ADDITIONAL_CA_KEY_ENV), "{error}"); + let error = TestCa::from_vars(|_| None).err().unwrap(); + assert!( + error.to_string().contains(ADDITIONAL_CA_CERT_ENV), + "{error}" + ); + } + + #[test] + fn leaf_needs_a_name() { + assert!(TestCa::new().unwrap().leaf(&[]).is_err()); + } +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 4f38bdb8f3..11ac2606f4 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -105,7 +105,8 @@ impl Drop for WebSocketRecorder { async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream, &mut Vec::new()).await else { + let local_addr = stream.local_addr().ok(); + let Some(request) = read_request(&mut stream, local_addr, &mut Vec::new()).await else { return; }; let Some(key) = header(&request.head, "sec-websocket-key") else { From b470b8558fb0b3c4bf9cb612cd06fbd6c65e40da Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:30:33 -0800 Subject: [PATCH 54/63] test(testkit): let a host HTTP server listen on a chosen port Signed-off-by: Matthew Grossman --- tests/support/README.md | 5 +- tests/support/host-server/src/http.rs | 89 ++++++++++++++++++++-- tests/support/host-server/src/lib.rs | 18 +++-- tests/support/host-server/src/tcp.rs | 2 +- tests/support/host-server/src/websocket.rs | 2 +- 5 files changed, 101 insertions(+), 15 deletions(-) diff --git a/tests/support/README.md b/tests/support/README.md index c13ba758e1..a2c0a02eaa 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -39,6 +39,7 @@ hooks and crate test settings. When a test must listen on an address the environment picked, because the gateway or another service is configured to call it, the environment -publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it. The -test never picks the address itself. Tests that bind the same address can't +publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it, for +example with `HttpRecorder::builder().port(port)` from `host-server-support`. +The test never picks the address itself. Tests that bind the same address can't run at the same time. diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index 8b1f830faa..ce7d62110e 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -173,14 +173,23 @@ impl HttpRecorder { pub async fn serve( handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - Self::listen(None, handler).await + Self::builder().serve(handler).await + } + + /// A recorder on a chosen port or with TLS, such as on a port the + /// environment published: + /// + /// ```ignore + /// let upstream = HttpRecorder::builder().port(port).tls(&identity).start("ok").await?; + /// ``` + pub fn builder() -> HttpRecorderBuilder { + HttpRecorderBuilder::default() } /// Answer every request over TLS with `200` and `body`, presenting /// `identity`'s certificate. pub async fn start_tls(identity: &Identity, body: impl Into) -> Result { - let body = body.into(); - Self::serve_tls(identity, move |_| Response::text(body.clone())).await + Self::builder().tls(identity).start(body).await } /// Answer every request over TLS with `handler`'s response, presenting @@ -190,14 +199,15 @@ impl HttpRecorder { identity: &Identity, handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - Self::listen(Some(identity.acceptor()?), handler).await + Self::builder().tls(identity).serve(handler).await } async fn listen( + port: u16, tls: Option, handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(port).await?; let handler: Arc = Arc::new(handler); let requests = Arc::new(Mutex::new(Vec::new())); let connections = Arc::new(AtomicUsize::new(0)); @@ -266,6 +276,47 @@ impl HttpRecorder { } } +/// Options for an [`HttpRecorder`], from [`HttpRecorder::builder`]. +#[derive(Debug, Default)] +pub struct HttpRecorderBuilder { + port: u16, + tls: Option, +} + +impl HttpRecorderBuilder { + /// Listen on `port` instead of a random free one. Fails to start if + /// something already listens there. + #[must_use] + pub fn port(mut self, port: u16) -> Self { + self.port = port; + self + } + + /// Serve TLS, presenting `identity`'s certificate. A connection that + /// fails the TLS handshake counts in + /// [`connections`](HttpRecorder::connections) but records no request. + #[must_use] + pub fn tls(mut self, identity: &Identity) -> Self { + self.tls = Some(identity.clone()); + self + } + + /// Answer every request with `200` and `body`. + pub async fn start(self, body: impl Into) -> Result { + let body = body.into(); + self.serve(move |_| Response::text(body.clone())).await + } + + /// Answer every request with `handler`'s response. + pub async fn serve( + self, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + let tls = self.tls.as_ref().map(Identity::acceptor).transpose()?; + HttpRecorder::listen(self.port, tls, handler).await + } +} + impl Drop for HttpRecorder { fn drop(&mut self) { self.task.abort(); @@ -442,6 +493,34 @@ mod tests { ); } + #[tokio::test] + async fn http_recorder_listens_on_a_chosen_port() { + let port = crate::free_port().await.unwrap(); + let recorder = HttpRecorder::builder() + .port(port) + .start("fixed") + .await + .unwrap(); + assert_eq!(recorder.port, port); + let mut stream = connect(port).await; + stream + .write_all(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n") + .await + .unwrap(); + assert!(read_to_end(&mut stream).await.ends_with("fixed")); + + let error = HttpRecorder::builder() + .port(port) + .start("taken") + .await + .err() + .unwrap(); + assert!( + error.to_string().contains(&format!("port {port}")), + "{error}" + ); + } + #[tokio::test] async fn http_recorder_keeps_a_connection_alive_until_the_request_closes_it() { let recorder = diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 374d3be419..b369cfac0c 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -12,7 +12,8 @@ //! //! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's //! [`Response`], and records each parsed [`Request`]. It serves HTTPS with -//! a certificate from a [`TestCa`]. +//! a certificate from a [`TestCa`], and listens on a random free port unless +//! [`HttpRecorder::builder`] names one. //! - [`WebSocketRecorder`] completes the WebSocket handshake and records the //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the @@ -30,7 +31,7 @@ use std::time::Duration; use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; -pub use http::{HttpRecorder, Request, Response}; +pub use http::{HttpRecorder, HttpRecorderBuilder, Request, Response}; pub use tcp::{TcpServer, exchange}; pub use tls::{ADDITIONAL_CA_CERT_ENV, ADDITIONAL_CA_KEY_ENV, Identity, TestCa}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; @@ -43,10 +44,15 @@ const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); const MAX_REQUEST: usize = 1024 * 1024; -async fn bind() -> Result<(TcpListener, u16)> { - let listener = TcpListener::bind(("0.0.0.0", 0)) +/// Listen on every host address at `port`, or at a random free port for 0. +async fn bind(port: u16) -> Result<(TcpListener, u16)> { + let listener = TcpListener::bind(("0.0.0.0", port)) .await - .context("bind a host server")?; + .context(if port == 0 { + "bind a host server".to_string() + } else { + format!("bind a host server on port {port}") + })?; let port = listener .local_addr() .context("read the host server's address")? @@ -56,7 +62,7 @@ async fn bind() -> Result<(TcpListener, u16)> { /// A port that nothing on the test host listened on when this returned. pub async fn free_port() -> Result { - let (_listener, port) = bind().await?; + let (_listener, port) = bind(0).await?; Ok(port) } diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs index 1b6d566895..f0a93ef006 100644 --- a/tests/support/host-server/src/tcp.rs +++ b/tests/support/host-server/src/tcp.rs @@ -45,7 +45,7 @@ impl TcpServer { } async fn start(mode: Mode) -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(0).await?; let received = Arc::new(Mutex::new(Vec::new())); let connections = Arc::new(AtomicUsize::new(0)); let task_received = Arc::clone(&received); diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 11ac2606f4..5488006e83 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -34,7 +34,7 @@ struct WebSocketState { impl WebSocketRecorder { pub async fn start() -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(0).await?; let state = Arc::new(WebSocketState::default()); let task_state = Arc::clone(&state); let task = tokio::spawn(async move { From 41184f2d6995900dc1536d6a2f92e166b19d9e01 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:45:34 -0800 Subject: [PATCH 55/63] test(tmachine): remove the Podman userns overrides block in default mode Signed-off-by: Matthew Grossman --- TESTING.md | 3 +- .../playbooks/drivers/podman/userns.yaml | 4 +-- .../gateway_overrides/defaults/main.yaml | 1 + .../roles/gateway_overrides/tasks/main.yaml | 30 +++++++++++++++++-- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/TESTING.md b/TESTING.md index 24e5fe6f05..6abd98c033 100644 --- a/TESTING.md +++ b/TESTING.md @@ -615,7 +615,8 @@ which takes a name and the tables to change, such as `[openshell.gateway]` or The role writes each name as its own marked block in `/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's -settings. Two blocks must not declare the same table. Don't write the file +settings, and `gateway_overrides_state: absent` removes a name's block. +Two blocks must not declare the same table. Don't write the file directly. The file is not a complete configuration. Each installer applies it to the gateway's own file: diff --git a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml index dc74fe8a93..feb3b3c323 100644 --- a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml +++ b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml @@ -6,7 +6,7 @@ # `podman_userns_mode` is default, auto, keep-id or private. Every mode except # default adds its `[openshell.drivers.podman]` settings to the gateway # overrides (the gateway_overrides role), which every installer applies to the -# gateway's configuration. Default mode adds nothing, so userns, uidmap and +# gateway's configuration. Default mode removes them, so userns, uidmap and # gidmap stay unset. The Podman support crate (tests/support/podman) reads the # published mode and runs Podman's own reference container in it. --- @@ -47,7 +47,7 @@ vars: gateway_overrides_name: podman-userns gateway_overrides_content: "{{ podman_userns_settings[podman_userns_mode] }}" - when: podman_userns_mode != "default" + gateway_overrides_state: "{{ 'absent' if podman_userns_mode == 'default' else 'present' }}" - name: Publish Podman user-namespace test variables ansible.builtin.include_tasks: ../../test-env.yaml diff --git a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml index 73e224fb07..8e9f90608e 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml @@ -3,3 +3,4 @@ --- gateway_overrides_path: /etc/openshell-test/gateway-overrides.toml +gateway_overrides_state: present diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml index 5114e90e69..f4d73a98f8 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -11,6 +11,8 @@ # the same name again replaces its block # gateway_overrides_content whole TOML tables, such as # `[openshell.drivers.podman]` +# gateway_overrides_state `absent` removes the name's block and keeps the +# others; the default is `present` # # Each name gets its own marked block, so writers keep each other's settings # and may run in any order. Two blocks must not declare the same table: TOML @@ -24,14 +26,19 @@ that: - gateway_overrides_name is defined - gateway_overrides_name is match('^[a-z0-9][a-z0-9-]*$') - - gateway_overrides_content is defined - - gateway_overrides_content | trim | length > 0 + - gateway_overrides_state in ['present', 'absent'] + - >- + gateway_overrides_state == 'absent' or + (gateway_overrides_content is defined and + gateway_overrides_content | trim | length > 0) fail_msg: >- gateway_overrides needs gateway_overrides_name (lowercase letters, - digits and dashes) and non-empty gateway_overrides_content + digits and dashes), gateway_overrides_state present or absent, and + non-empty gateway_overrides_content unless the state is absent - name: Create the test configuration directory become: true + when: gateway_overrides_state == 'present' ansible.builtin.file: path: "{{ gateway_overrides_path | dirname }}" state: directory @@ -49,3 +56,20 @@ mode: "0644" marker: "# {mark} {{ gateway_overrides_name }}" block: "{{ gateway_overrides_content }}" + when: gateway_overrides_state == 'present' + +- name: Check for the gateway overrides file + ansible.builtin.stat: + path: "{{ gateway_overrides_path }}" + register: gateway_overrides_file + when: gateway_overrides_state == 'absent' + +- name: Remove the {{ gateway_overrides_name }} gateway overrides + become: true + ansible.builtin.blockinfile: + path: "{{ gateway_overrides_path }}" + marker: "# {mark} {{ gateway_overrides_name }}" + state: absent + when: + - gateway_overrides_state == 'absent' + - gateway_overrides_file.stat.exists From d8aa49b0b6cbd56a60558c404b2eeda1de5d65e4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:45:34 -0800 Subject: [PATCH 56/63] test(testkit): reject a mismatched additional CA key at load time Signed-off-by: Matthew Grossman --- tests/support/README.md | 6 ++++-- tests/support/host-server/src/tls.rs | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/support/README.md b/tests/support/README.md index a2c0a02eaa..44cf957511 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -12,7 +12,9 @@ tests to read. A tmachine environment publishes its variables with `tests/tmachine/ansible/playbooks/test-env.yaml`, and `openshell-test-runner` passes them to the tests. To run a suite against your own deployment, set them yourself. A fixture fails the test when a variable it needs is missing; it -never skips it. Add a row when you add a variable. +never skips it. Add a row when you add a variable. The runner also sets +`OPENSHELL_GATEWAY` and `OPENSHELL_BIN`, which testkit reads; see +[Running against a gateway](../testkit/README.md#running-against-a-gateway). | Variable | Set by | Read by | Meaning | |---|---|---|---| @@ -23,7 +25,7 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | -| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | +| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which sets the driver's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the supervisor trusts for sandbox egress. | | `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | | `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs index 6b36293b92..a379b35cc3 100644 --- a/tests/support/host-server/src/tls.rs +++ b/tests/support/host-server/src/tls.rs @@ -9,7 +9,7 @@ //! - [`TestCa::new`] makes a CA that only the test knows, so nothing trusts it //! unless the test hands over [`TestCa::cert_pem`]. //! - [`TestCa::from_env`] loads the environment's additional CA, which the -//! gateway is configured to trust (`additional_ca_bundle`). The environment +//! supervisor trusts for sandbox egress (`additional_ca_bundle`). The environment //! publishes the paths of its PEM certificate and key in //! `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY`. @@ -82,9 +82,23 @@ impl TestCa { "parse the CA certificate {}", cert.display() )))?; - let key = KeyPair::from_pem(&key_pem) + let key_pair = KeyPair::from_pem(&key_pem) .map_err(failed(format!("parse the CA key {}", key.display())))?; - let issuer = pki::issuer_from_der(&der, key) + // issuer_from_der reads the certificate's metadata only, so check the + // key here rather than at the first client handshake. + let key_der = key_pair + .serialize_der() + .map_err(failed(format!("export the CA key {}", key.display())))?; + openshell_crypto::tls::try_server_builder() + .map_err(failed("build a TLS server configuration"))? + .with_no_client_auth() + .with_single_cert(vec![der.clone()], PrivateKeyDer::Pkcs8(key_der.into())) + .map_err(failed(format!( + "the CA key {} does not match the CA certificate {}", + key.display(), + cert.display() + )))?; + let issuer = pki::issuer_from_der(&der, key_pair) .map_err(failed(format!("load the CA {}", cert.display())))?; Ok(Self { cert_pem, issuer }) } @@ -301,6 +315,14 @@ mod tests { ); } + #[test] + fn from_files_rejects_a_key_from_another_ca() { + let (cert, _, _) = write_ca("host-server-mismatch-a"); + let (_, other_key, _) = write_ca("host-server-mismatch-b"); + let error = TestCa::from_files(&cert, &other_key).err().unwrap(); + assert!(error.to_string().contains("does not match"), "{error}"); + } + #[test] fn leaf_needs_a_name() { assert!(TestCa::new().unwrap().leaf(&[]).is_err()); From 8b1fcef6e628ab01d8e97829ff7398a4a16832c4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:48:58 -0800 Subject: [PATCH 57/63] test(tmachine): require a table header in gateway overrides and skip an empty file Signed-off-by: Matthew Grossman --- .../tmachine/ansible/roles/gateway_overrides/tasks/main.yaml | 5 +++-- .../roles/openshell_gateway/tasks/merge-settings.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml index f4d73a98f8..5da35a936f 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -30,11 +30,12 @@ - >- gateway_overrides_state == 'absent' or (gateway_overrides_content is defined and - gateway_overrides_content | trim | length > 0) + gateway_overrides_content | trim is match('\\[')) fail_msg: >- gateway_overrides needs gateway_overrides_name (lowercase letters, digits and dashes), gateway_overrides_state present or absent, and - non-empty gateway_overrides_content unless the state is absent + gateway_overrides_content that starts with a table header, such as + `[openshell.gateway]`, unless the state is absent - name: Create the test configuration directory become: true diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml index 2b3b8e3e22..05a679b2f5 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml @@ -28,4 +28,4 @@ {{ '--default ' ~ openshell_gateway_merge_default if openshell_gateway_merge_default | default('') else '' }} executable: /usr/bin/python3 - when: openshell_gateway_overrides.stat.exists + when: openshell_gateway_overrides.stat.exists and openshell_gateway_overrides.stat.size > 0 From 8005863c691ac996ed118934041bca7508b48ff5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:48:58 -0800 Subject: [PATCH 58/63] test(testkit): harden host server TLS and name environment ports _PORT Add the authority key identifier to server certificates, which strict verifiers such as Python 3.13 require, time out TLS handshakes, report CA load failures as such, and keep test CA files in temporary directories. The contract table now names environment-chosen ports OPENSHELL_TEST__PORT, matching HttpRecorder's port option. Signed-off-by: Matthew Grossman --- tests/support/Cargo.lock | 1 + tests/support/README.md | 14 +++++------ tests/support/host-server/Cargo.toml | 1 + tests/support/host-server/src/http.rs | 3 ++- tests/support/host-server/src/tls.rs | 36 ++++++++++++++++++++------- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 440b9ec84b..7eb55a6e94 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1286,6 +1286,7 @@ dependencies = [ "rcgen", "rustls", "serde_json", + "tempfile", "tokio", "tokio-rustls", "tungstenite", diff --git a/tests/support/README.md b/tests/support/README.md index 44cf957511..80402e79b9 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -37,11 +37,11 @@ purpose: unit-test child-process markers, the re-entry markers in `tests/support/environment.rs`, Nix guest-cache settings, `install.sh` test hooks and crate test settings. -### Addresses the environment chooses +### Ports the environment chooses -When a test must listen on an address the environment picked, because the -gateway or another service is configured to call it, the environment -publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it, for -example with `HttpRecorder::builder().port(port)` from `host-server-support`. -The test never picks the address itself. Tests that bind the same address can't -run at the same time. +When a test must listen on a port the environment picked, because the gateway +or another service is configured to call it, the environment publishes it as +`OPENSHELL_TEST__PORT` and the test binds that port on every host +address, for example with `HttpRecorder::builder().port(port)` from +`host-server-support`. The test never picks the port itself. Tests that bind +the same port can't run at the same time. diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index ccac94f0d1..0b91888a99 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -22,4 +22,5 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["tls12" tungstenite = "0.26" [dev-dependencies] +tempfile = "3" tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index ce7d62110e..c8804deab7 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -225,7 +225,8 @@ impl HttpRecorder { match tls { None => respond(stream, local_addr, handler, requests).await, Some(tls) => { - if let Ok(stream) = tls.accept(stream).await { + let handshake = tokio::time::timeout(READ_TIMEOUT, tls.accept(stream)); + if let Ok(Ok(stream)) = handshake.await { respond(stream, local_addr, handler, requests).await; } } diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs index a379b35cc3..1102c306ae 100644 --- a/tests/support/host-server/src/tls.rs +++ b/tests/support/host-server/src/tls.rs @@ -94,7 +94,7 @@ impl TestCa { .with_no_client_auth() .with_single_cert(vec![der.clone()], PrivateKeyDer::Pkcs8(key_der.into())) .map_err(failed(format!( - "the CA key {} does not match the CA certificate {}", + "load the CA key {} with the CA certificate {}", key.display(), cert.display() )))?; @@ -121,6 +121,8 @@ impl TestCa { params.distinguished_name.push(DnType::CommonName, first); params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + // Strict verifiers, such as Python 3.13 and later, require it. + params.use_authority_key_identifier_extension = true; let key = pki::generate_keypair().map_err(failed("generate a server key"))?; let cert = pki::signed_by_issuer(params, &key, &self.issuer) .map_err(failed(format!("sign a server certificate for {names:?}")))?; @@ -192,6 +194,7 @@ mod tests { use std::path::PathBuf; use rustls::pki_types::ServerName; + use tempfile::TempDir; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::TcpStream; use tokio_rustls::TlsConnector; @@ -231,12 +234,10 @@ mod tests { String::from_utf8_lossy(&response).into_owned() } - fn write_ca(dir: &str) -> (PathBuf, PathBuf, String) { + fn write_ca(dir: &TempDir) -> (PathBuf, PathBuf, String) { let key = pki::generate_keypair().unwrap(); let cert = pki::self_signed(ca_params(), &key).unwrap(); - let dir = std::env::temp_dir().join(format!("{dir}-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let (cert_path, key_path) = (dir.join("ca.pem"), dir.join("ca-key.pem")); + let (cert_path, key_path) = (dir.path().join("ca.pem"), dir.path().join("ca-key.pem")); std::fs::write(&cert_path, cert.pem()).unwrap(); std::fs::write(&key_path, key.serialize_pem().unwrap()).unwrap(); (cert_path, key_path, cert.pem()) @@ -283,7 +284,8 @@ mod tests { #[tokio::test] async fn from_vars_loads_the_environment_ca() { - let (cert, key, cert_pem) = write_ca("host-server-env-ca"); + let dir = TempDir::new().unwrap(); + let (cert, key, cert_pem) = write_ca(&dir); let ca = TestCa::from_vars(|name| match name { ADDITIONAL_CA_CERT_ENV => Some(cert.display().to_string()), ADDITIONAL_CA_KEY_ENV => Some(key.display().to_string()), @@ -317,10 +319,26 @@ mod tests { #[test] fn from_files_rejects_a_key_from_another_ca() { - let (cert, _, _) = write_ca("host-server-mismatch-a"); - let (_, other_key, _) = write_ca("host-server-mismatch-b"); + let (a, b) = (TempDir::new().unwrap(), TempDir::new().unwrap()); + let (cert, _, _) = write_ca(&a); + let (_, other_key, _) = write_ca(&b); let error = TestCa::from_files(&cert, &other_key).err().unwrap(); - assert!(error.to_string().contains("does not match"), "{error}"); + assert!( + error.to_string().contains("with the CA certificate"), + "{error}" + ); + } + + #[test] + fn leaf_names_its_issuer_key() { + const AUTHORITY_KEY_IDENTIFIER: &[u8] = &[0x06, 0x03, 0x55, 0x1d, 0x23]; + let leaf = TestCa::new().unwrap().leaf(&["localhost"]).unwrap(); + assert!( + leaf.cert_der + .windows(AUTHORITY_KEY_IDENTIFIER.len()) + .any(|window| window == AUTHORITY_KEY_IDENTIFIER), + "the server certificate has no authority key identifier" + ); } #[test] From e780266ed23382d41018d96222cb03a72b27f300 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 23:05:06 -0800 Subject: [PATCH 59/63] fix(tmachine): give the OIDC gateway explicit TLS paths for config preflight Signed-off-by: Matthew Grossman --- .../ansible/roles/oidc_gateway/defaults/main.yaml | 2 ++ .../tmachine/ansible/roles/oidc_gateway/tasks/main.yaml | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml b/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml index 1e20e5a1da..14a375559c 100644 --- a/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml +++ b/tests/tmachine/ansible/roles/oidc_gateway/defaults/main.yaml @@ -3,3 +3,5 @@ --- oidc_gateway_operator_script: /usr/local/libexec/openshell-test/operator-client-cert +# OPENSHELL_LOCAL_TLS_DIR in the packaged gateway's unit. +oidc_gateway_tls_dir: "{{ openshell_gateway_home }}/.local/state/openshell/tls" diff --git a/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml b/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml index 1a7666c7ca..84525eed3f 100644 --- a/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/oidc_gateway/tasks/main.yaml @@ -36,6 +36,15 @@ [openshell.gateway.mtls_auth] operator_enabled = true + # `config preflight` accepts operator authentication only with an + # explicit client CA; it doesn't use the local TLS files the packaged + # gateway otherwise finds on its own. These are the paths + # `generate-certs` writes to. + [openshell.gateway.tls] + cert_path = "{{ oidc_gateway_tls_dir }}/server/tls.crt" + key_path = "{{ oidc_gateway_tls_dir }}/server/tls.key" + client_ca_path = "{{ oidc_gateway_tls_dir }}/ca.crt" + - name: Create the test scripts directory become: true ansible.builtin.file: From 6740fa350030327672bd512f8d37e063a924e123 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 23:54:15 -0800 Subject: [PATCH 60/63] fix(oidc-auth): recognize the operator by its operator role Signed-off-by: Matthew Grossman --- .../features/oidc-auth/tests/oidc_auth.rs | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/suites/features/oidc-auth/tests/oidc_auth.rs b/tests/suites/features/oidc-auth/tests/oidc_auth.rs index 4433660168..3818fd0c39 100644 --- a/tests/suites/features/oidc-auth/tests/oidc_auth.rs +++ b/tests/suites/features/oidc-auth/tests/oidc_auth.rs @@ -730,22 +730,28 @@ async fn user_can_resolve_provider_profile_for_sandbox(cx: &mut Context) -> Resu /// operator, which the tests' setup and cleanup need. async fn setup(cx: &mut Context) -> Result> { let kc = KeycloakContext::new(cx).await?; - let operator = kc.cli_handle().gateway().connect().await?; - match operator + let me = kc + .cli_handle() + .gateway() + .connect() + .await? .raw_grpc_fresh() .await? .get_current_user(proto::GetCurrentUserRequest {}) .await - { - Err(status) if status.code() == GrpcCode::PermissionDenied => Ok(kc), - Err(status) => Err(Error::from(status).context("check testkit connects as an operator")), - Ok(user) => Err(Error::msg(format!( - "testkit's client certificate signs in as user {:?} ({}), not as an operator; \ - the -oidc environment makes it an operator's (roles/oidc_gateway)", - user.get_ref().subject, - user.get_ref().identity_provider - ))), - } + .context("check testkit connects as an operator")? + .into_inner(); + // The gateway gives an operator certificate's identity the `operator` + // role; any other certificate signs in as an ordinary user. + testkit::ensure!( + me.identity_provider == "mtls" && me.roles.iter().any(|role| role == "operator"), + "testkit's client certificate signs in as {:?} ({}, roles {:?}), not as an operator; \ + the -oidc environment makes it an operator's (roles/oidc_gateway)", + me.subject, + me.identity_provider, + me.roles + ); + Ok(kc) } async fn raw(session: &Session) -> Result { From 82b5c068a915389ad88d4e7ee0050b3076c386cb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 20 Nov 2026 00:38:37 -0800 Subject: [PATCH 61/63] fix(oidc-auth): create providers against the workspace's imported profile Signed-off-by: Matthew Grossman --- tests/suites/features/oidc-auth/tests/oidc_auth.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/suites/features/oidc-auth/tests/oidc_auth.rs b/tests/suites/features/oidc-auth/tests/oidc_auth.rs index 3818fd0c39..1de13161e8 100644 --- a/tests/suites/features/oidc-auth/tests/oidc_auth.rs +++ b/tests/suites/features/oidc-auth/tests/oidc_auth.rs @@ -835,6 +835,7 @@ fn provider_request(workspace: &str, name: &str, profile: &str) -> proto::Create ..Default::default() }), r#type: profile.to_string(), + profile_workspace: workspace.to_string(), credentials: [(CREDENTIAL_ENV.to_string(), CREDENTIAL_VALUE.to_string())].into(), ..Default::default() }), From c3ec078afc2d34e26622eeeb43d288610e63151e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 20 Nov 2026 08:25:50 -0800 Subject: [PATCH 62/63] test(continuity): add prost-types to the lockfile for testkit Signed-off-by: Matthew Grossman --- tests/suites/continuity/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/suites/continuity/Cargo.lock b/tests/suites/continuity/Cargo.lock index 26179cb602..234f054fe9 100644 --- a/tests/suites/continuity/Cargo.lock +++ b/tests/suites/continuity/Cargo.lock @@ -2407,6 +2407,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", From 5a9993e95b5729f2a8da4faa700403980d428372 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Fri, 20 Nov 2026 08:57:15 -0800 Subject: [PATCH 63/63] chore(continuity): refresh the lockfile for testkit's new dependencies Signed-off-by: Matthew Grossman --- tests/suites/continuity/Cargo.lock | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/suites/continuity/Cargo.lock b/tests/suites/continuity/Cargo.lock index 234f054fe9..7a26c8a821 100644 --- a/tests/suites/continuity/Cargo.lock +++ b/tests/suites/continuity/Cargo.lock @@ -2356,6 +2356,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2404,12 +2419,16 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "prost-types", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile",