From 3e4572196ffd32cd4326f07428c5225fd35d6f4a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:29:33 -0800 Subject: [PATCH 01/10] test(testkit): let create_provider set provider config create_provider and install_provider return a builder that creates the provider when awaited, so tests can chain .config(key, value) for provider create --config. Existing callers are unchanged. Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 3 +- tests/testkit/testkit/src/lib.rs | 1 + tests/testkit/testkit/src/provider.rs | 140 +++++++++++++++++++------- 3 files changed, 109 insertions(+), 35 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 9cc6120620..b136777dc3 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -115,7 +115,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a provider, each with cleanup. `cx.install_provider(name, profile, - credentials)` does both. + credentials)` does both. Chain `.config(key, value)` before the `.await` + to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7c233ce834..5772ac52fd 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -58,6 +58,7 @@ pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; +pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index b68b20989c..3cd6a89114 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::future::{Future, IntoFuture}; +use std::pin::Pin; + use serde::Deserialize; use crate::context::Context; @@ -31,57 +34,126 @@ impl Context { } /// Create a provider called `name` of `provider_type` with `credentials`, - /// as pairs of environment variable and value. Cleanup deletes the - /// provider after the test's sandboxes. + /// as pairs of environment variable and value, when awaited. Cleanup + /// deletes the provider after the test's sandboxes. /// /// The CLI reads the credential values from its environment, so they - /// aren't in the reported command. - pub async fn create_provider( - &self, - name: &str, - provider_type: &str, - credentials: &[(&str, &str)], - ) -> Result { + /// aren't in the reported command. [`ProviderBuilder`] sets other + /// `provider create` options before the await. + pub fn create_provider<'a>( + &'a self, + name: &'a str, + provider_type: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Named(provider_type), credentials) + } + + /// Import `profile` with [`Context::import_profile`] and create a provider + /// called `name` of its type with [`Context::create_provider`], when + /// awaited. + pub fn install_provider<'a>( + &'a self, + name: &'a str, + profile: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Profile(profile), credentials) + } + + fn on_cleanup_delete(&self, kind: &'static str, name: &str) { + let cli = self.cli_handle(); + let owned = name.to_string(); + self.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } +} + +/// Where a [`ProviderBuilder`] gets its provider type. +enum ProviderType<'a> { + /// A profile already on the gateway. + Named(&'a str), + /// A profile, in YAML, to import first. + Profile(&'a str), +} + +/// A provider to create, from [`Context::create_provider`] or +/// [`Context::install_provider`]. Awaiting it creates the provider. +/// +/// ```ignore +/// cx.create_provider(&name, &provider_type, &[]) +/// .config("project", "production") +/// .await?; +/// ``` +#[must_use = "a ProviderBuilder does nothing until awaited"] +pub struct ProviderBuilder<'a> { + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + config: Vec, +} + +impl<'a> ProviderBuilder<'a> { + fn new( + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + ) -> Self { + Self { + cx, + name, + provider_type, + credentials, + config: Vec::new(), + } + } + + /// Set the provider config `key` to `value` (`--config key=value`). + pub fn config(mut self, key: &str, value: &str) -> Self { + self.config.push(format!("{key}={value}")); + self + } + + async fn create(self) -> Result { + let provider_type = match self.provider_type { + ProviderType::Named(provider_type) => provider_type.to_string(), + ProviderType::Profile(profile) => self.cx.import_profile(profile).await?, + }; // Deleting a missing provider succeeds, so register the delete before // the create. - self.on_cleanup_delete("provider", name); + self.cx.on_cleanup_delete("provider", self.name); let mut args = vec![ "provider", "create", "--name", - name, + self.name, "--type", - provider_type, + &provider_type, ]; - for (key, _) in credentials { + for (key, _) in self.credentials { args.extend(["--credential", key]); } - self.cli_with_env(credentials, args) + for setting in &self.config { + args.extend(["--config", setting]); + } + self.cx + .cli_with_env(self.credentials, args) .await? .success() - .context(format!("create provider {name}"))?; + .context(format!("create provider {}", self.name))?; Ok(()) } +} - /// Import `profile` with [`Context::import_profile`] and create a provider - /// called `name` of its type with [`Context::create_provider`]. - pub async fn install_provider( - &self, - name: &str, - profile: &str, - credentials: &[(&str, &str)], - ) -> Result { - let provider_type = self.import_profile(profile).await?; - self.create_provider(name, &provider_type, credentials) - .await - } +impl<'a> IntoFuture for ProviderBuilder<'a> { + type Output = Result; + type IntoFuture = Pin + 'a>>; - fn on_cleanup_delete(&self, kind: &'static str, name: &str) { - let cli = self.cli_handle(); - let owned = name.to_string(); - self.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &owned]).await?.success()?; - Ok(()) - }); + fn into_future(self) -> Self::IntoFuture { + Box::pin(self.create()) } } From 1440a1da99b84b5853f821f2d5c0490b31b6800d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:31:38 -0800 Subject: [PATCH 02/10] test(conformance): port provider files and auto-create tests to testkit Move provider_files and provider_auto_create from e2e/rust into a providers conformance suite. The auto-create test imports a run-unique profile instead of relying on a globally imported claude-code profile, so it no longer changes shared gateway state. Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 5 - e2e/rust/tests/provider_auto_create.rs | 197 --------------------- e2e/rust/tests/provider_files.rs | 176 ------------------ tests/suites/conformance/README.md | 1 + tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/providers.rs | 206 ++++++++++++++++++++++ tests/tmachine/artifacts.nix | 3 - 7 files changed, 208 insertions(+), 381 deletions(-) delete mode 100644 e2e/rust/tests/provider_auto_create.rs delete mode 100644 e2e/rust/tests/provider_files.rs create mode 100644 tests/suites/conformance/src/providers.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index c35f4ffea6..a70df6d3c4 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -61,11 +61,6 @@ name = "policy_activation" path = "tests/policy_activation.rs" required-features = ["e2e-docker"] -[[test]] -name = "provider_files" -path = "tests/provider_files.rs" -required-features = ["e2e-docker"] - [[test]] name = "oidc_pkce" path = "tests/oidc_pkce.rs" diff --git a/e2e/rust/tests/provider_auto_create.rs b/e2e/rust/tests/provider_auto_create.rs deleted file mode 100644 index 2c8828deaf..0000000000 --- a/e2e/rust/tests/provider_auto_create.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: `--provider ` auto-creates a provider from local credentials. -//! -//! When `--provider claude-code` is passed and no provider named "claude-code" exists, -//! the CLI should discover `ANTHROPIC_API_KEY` from the local environment, -//! auto-create a provider, and inject a supervisor-managed placeholder into the -//! sandbox child process environment. -//! -//! The sandbox command (`printenv ANTHROPIC_API_KEY`) verifies that the -//! placeholder made it all the way through to the sandbox process environment. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::process::Stdio; -use std::sync::Mutex; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -#[cfg(feature = "e2e-local-container-driver")] -use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; - -const TEST_API_KEY: &str = "sk-e2e-auto-provider-test-key"; -static CLAUDE_PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -fn contains_placeholder_for_env_key(output: &str, key: &str) -> bool { - let legacy = format!("openshell:resolve:env:{key}"); - let revision_prefix = "openshell:resolve:env:v"; - let revision_suffix = format!("_{key}"); - output.split_whitespace().any(|token| { - token == legacy || (token.starts_with(revision_prefix) && token.ends_with(&revision_suffix)) - }) -} - -/// Helper: delete a provider by name, ignoring errors. -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// Helper: check whether a provider already exists. -async fn provider_exists(name: &str) -> bool { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("get") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - cmd.status().await.is_ok_and(|status| status.success()) -} - -/// Helper: delete a sandbox by name, ignoring errors. -async fn delete_sandbox(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("sandbox") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// `--provider claude-code --auto-providers` with `ANTHROPIC_API_KEY` set should -/// auto-create a "claude-code" provider and inject a placeholder into the sandbox. -#[tokio::test] -async fn auto_created_provider_credential_available_in_sandbox() { - let _provider_lock = CLAUDE_PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - if provider_exists("claude-code").await { - eprintln!("Skipping test: existing provider 'claude-code' would make shared state unsafe"); - return; - } - - // Clean up any leftover from a previous run. - delete_provider("claude-code").await; - - // This test only reads the injected environment placeholder. Do not inherit - // the published image's network rules, which may be incompatible with the - // attached credential provider's startup validation. - let policy = tempfile::NamedTempFile::new().expect("create provider test policy"); - std::fs::write( - policy.path(), - r"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /etc, /proc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: {} -", - ) - .expect("write provider test policy"); - - // Create a sandbox that prints the ANTHROPIC_API_KEY env var. - // --auto-providers skips the interactive prompt. - let mut cmd = openshell_cmd(); - cmd.arg("sandbox").arg("create"); - #[cfg(feature = "e2e-local-container-driver")] - cmd.arg("--from").arg(E2E_WORKLOAD_IMAGE); - cmd.arg("--detach") - .arg("--policy") - .arg(policy.path()) - .arg("--provider") - .arg("claude-code") - .arg("--auto-providers") - .args(["--", "sh", "-c", "exec sleep infinity"]) - .env("ANTHROPIC_API_KEY", TEST_API_KEY) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .expect("failed to spawn openshell sandbox create"); - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let clean = strip_ansi(&combined); - - // Parse sandbox name for cleanup. - let sandbox_name = extract_field(&combined, "Created sandbox"); - let exec_output = if let Some(ref name) = sandbox_name { - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - name, - "--no-tty", - "--", - "printenv", - "ANTHROPIC_API_KEY", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - Some(exec_cmd.output().await.expect("failed to run sandbox exec")) - } else { - None - }; - - // Always clean up, even if assertions fail. - if let Some(ref name) = sandbox_name { - delete_sandbox(name).await; - } - delete_provider("claude-code").await; - - // Now assert. - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{clean}", - output.status.code() - ); - - let exec_output = exec_output.expect("sandbox name should be present"); - let exec_clean = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&exec_output.stdout), - String::from_utf8_lossy(&exec_output.stderr) - )); - assert!( - exec_output.status.success(), - "sandbox exec should succeed:\n{exec_clean}" - ); - - assert!( - clean.contains("Created provider claude-code"), - "output should confirm provider auto-creation:\n{clean}" - ); - - assert!( - contains_placeholder_for_env_key(&exec_clean, "ANTHROPIC_API_KEY"), - "sandbox should have placeholder ANTHROPIC_API_KEY in its environment:\n{exec_clean}" - ); - - assert!( - !exec_clean.contains(TEST_API_KEY), - "sandbox should not expose the raw ANTHROPIC_API_KEY secret:\n{exec_clean}" - ); -} diff --git a/e2e/rust/tests/provider_files.rs b/e2e/rust/tests/provider_files.rs deleted file mode 100644 index 18db5db9b7..0000000000 --- a/e2e/rust/tests/provider_files.rs +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! A provider profile serves read-only config on each workload open. - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_bin; -use openshell_e2e::harness::cli::run_cli; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::sleep; - -struct ProviderGuard { - profile: String, - provider: String, -} - -impl Drop for ProviderGuard { - fn drop(&mut self) { - let binary = openshell_bin(); - for _ in 0..20 { - let deleted = std::process::Command::new(&binary) - .args(["provider", "delete", &self.provider]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - if deleted.is_ok_and(|status| status.success()) { - break; - } - std::thread::sleep(Duration::from_millis(250)); - } - let _ = std::process::Command::new(&binary) - .args(["profile", "delete", &self.profile]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn cli_ok(args: &[&str]) -> Result<(), String> { - let (output, code) = run_cli(args).await; - if code == 0 { - Ok(()) - } else { - Err(format!( - "{} failed (exit {code}):\n{output}", - args.join(" ") - )) - } -} - -#[tokio::test] -async fn provider_file_open_update_and_detach() -> Result<(), String> { - let suffix = format!("{}-{:08x}", std::process::id(), rand::random::()); - let profile_id = format!("pf-{suffix}"); - let provider = format!("pf-{suffix}"); - let profile_yaml = include_str!("../../../examples/provider-managed-files/acme-config.yaml") - .replace("id: acme-config", &format!("id: {profile_id}")); - let mut profile_file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| error.to_string())?; - profile_file - .write_all(profile_yaml.as_bytes()) - .map_err(|error| error.to_string())?; - cli_ok(&[ - "profile", - "import", - "--file", - profile_file - .path() - .to_str() - .ok_or("profile path is not UTF-8")?, - ]) - .await?; - let _provider_guard = ProviderGuard { - profile: profile_id.clone(), - provider: provider.clone(), - }; - cli_ok(&[ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--config", - "endpoint=https://api.acme.example", - "--config", - "project=production", - ]) - .await?; - - let mut sandbox = SandboxGuard::create(&["--provider", &provider, "--no-tty"]).await?; - let path = format!("/run/openshell/providers/{provider}/client.toml"); - let environment_path = sandbox.exec(&["printenv", "ACME_CONFIG_FILE"]).await?; - if !environment_path.contains(&path) { - return Err(format!( - "provider path environment variable missing: {environment_path}" - )); - } - let before = sandbox.exec(&["cat", &path]).await?; - if !before.contains("project = \"production\"") { - return Err(format!("initial provider file content missing:\n{before}")); - } - let permissions = sandbox - .exec(&[ - "sh", - "-c", - &format!( - "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ - test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ - if (printf x >&3) 2>/dev/null; then exit 1; fi; \ - echo provider-file-permissions-ok" - ), - ]) - .await?; - if !permissions.contains("provider-file-permissions-ok") { - return Err(format!( - "provider file permission assertion did not complete:\n{permissions}" - )); - } - - cli_ok(&[ - "provider", - "update", - &provider, - "--config", - "project=staging", - "--wait", - "--timeout", - "90", - ]) - .await?; - let after = sandbox.exec(&["cat", &path]).await?; - if !after.contains("project = \"staging\"") { - return Err(format!("updated provider file content missing:\n{after}")); - } - - cli_ok(&[ - "sandbox", - "provider", - "detach", - &sandbox.name, - &provider, - "--wait", - "--timeout", - "90", - ]) - .await?; - let (detached, code) = run_cli(&[ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - &path, - ]) - .await; - if code == 0 || !detached.contains("No such file or directory") { - return Err(format!( - "detached provider path remained readable: {detached}" - )); - } - sandbox.cleanup().await; - // Let the asynchronous sandbox deletion release the provider before - // ProviderGuard removes the provider record and its profile. - sleep(Duration::from_millis(250)).await; - Ok(()) -} diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 0b19227aa9..27eadb7c99 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -19,6 +19,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `create-upload` | `sandbox create --upload`: directory basename and nested files, multiple uploads, a single file | | `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `providers` | Provider profile files: path, mode, config update, detach; `sandbox create --auto-providers` creates a provider from local credentials and injects a placeholder | | `image` | A sandbox runs the default image or the one it is created from | | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 8ac23f54bc..c0aa355e5a 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -21,6 +21,7 @@ mod lifecycle; mod persistent_workspace; mod policy_advisor; mod process; +mod providers; mod settings; mod smoke; mod ssh; diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs new file mode 100644 index 0000000000..e67a349e3a --- /dev/null +++ b/tests/suites/conformance/src/providers.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Providers attached to a sandbox: files a provider profile serves, and +//! providers that `sandbox create --provider` creates from local credentials. + +use testkit::{Context, ResultExt as _, YamlFile, policy}; + +/// The example profile that serves `client.toml` from the provider's config. +const FILES_PROFILE: &str = + include_str!("../../../../examples/provider-managed-files/acme-config.yaml"); +const AUTO_CREATE_ENV: &str = "E2E_AUTO_PROVIDER_KEY"; +const AUTO_CREATE_SECRET: &str = "sk-e2e-auto-provider-test-key"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; + +/// A provider profile's file is readable at the path its environment variable +/// names, with mode 600 and no write access. Updating the provider's config +/// updates the file, and detaching the provider removes it. +#[testkit::test] +async fn provider_file_open_update_and_detach(cx: &mut Context) -> testkit::Result { + let provider = cx.name("pf"); + let profile = FILES_PROFILE.replace("id: acme-config", &format!("id: {provider}")); + cx.install_provider(&provider, &profile, &[]) + .config("endpoint", "https://api.acme.example") + .config("project", "production") + .await?; + + let sandbox = cx + .sandbox("pf") + .spec(|spec| spec.providers = vec![provider.clone()]) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let path = format!("/run/openshell/providers/{provider}/client.toml"); + sandbox + .exec(["printenv", "ACME_CONFIG_FILE"]) + .await? + .expect_stdout(&format!("{path}\n")) + .context("the provider file's environment variable")?; + expect_project(&sandbox, &path, "production").await?; + sandbox + .exec([ + "sh", + "-c", + &format!( + "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ + test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ + if (printf x >&3) 2>/dev/null; then exit 1; fi" + ), + ]) + .await? + .success() + .context("the provider file is a read-only regular file with mode 600")?; + + cx.cli([ + "provider", + "update", + &provider, + "--config", + "project=staging", + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + expect_project(&sandbox, &path, "staging").await?; + + cx.cli([ + "sandbox", + "provider", + "detach", + sandbox.name(), + &provider, + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + let detached = sandbox.exec(["cat", &path]).await?.failure()?; + testkit::ensure!( + detached.stderr.contains("No such file or directory"), + "the detached provider's file is still there: {}", + detached.stderr + ); + Ok(()) +} + +/// `sandbox create --provider --auto-providers` creates a +/// provider named after a profile from the credential in the CLI's +/// environment. The sandbox gets a placeholder for the credential, not the +/// secret. +#[testkit::test] +async fn auto_created_provider_credential_available_in_sandbox( + cx: &mut Context, +) -> testkit::Result { + let provider = cx + .import_profile(&auto_create_profile(&cx.name("auto"))) + .await?; + // The CLI names the provider it creates after the profile. + let cli = cx.cli_handle(); + let name = provider.clone(); + cx.on_cleanup(format!("delete provider {provider}"), async move || { + cli.run(["provider", "delete", &name]).await?.success()?; + Ok(()) + }); + + // The test only reads the environment. A policy without network rules + // keeps the default policy's rules from conflicting with the provider's. + let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; + let sandbox_name = cx.name("ac"); + cx.track(&sandbox_name); + let create = cx + .cli_with_env( + &[(AUTO_CREATE_ENV, AUTO_CREATE_SECRET)], + [ + "sandbox", + "create", + "--name", + &sandbox_name, + "--no-tty", + "--detach", + "--policy", + policy.path()?, + "--provider", + &provider, + "--auto-providers", + "--", + "sh", + "-c", + "exec sleep infinity", + ], + ) + .await? + .success()?; + testkit::ensure!( + create + .report_text() + .contains(&format!("Created provider {provider}")), + "sandbox create did not report creating provider {provider}:\n{}{}", + create.stdout, + create.stderr + ); + + let sandbox = cx.adopt(&sandbox_name).await?; + sandbox.wait_ready().await?; + let output = sandbox + .exec(["printenv", AUTO_CREATE_ENV]) + .await? + .success()?; + let value = output.stdout.trim(); + testkit::ensure!( + !value.contains(AUTO_CREATE_SECRET), + "the sandbox has the raw credential in {AUTO_CREATE_ENV}" + ); + testkit::ensure!( + is_placeholder_for(value, AUTO_CREATE_ENV), + "expected a placeholder for {AUTO_CREATE_ENV} in the sandbox; got {value:?}" + ); + Ok(()) +} + +async fn expect_project(sandbox: &testkit::Sandbox, path: &str, project: &str) -> testkit::Result { + let content = sandbox.exec(["cat", path]).await?.success()?.stdout; + testkit::ensure!( + content.contains(&format!("project = \"{project}\"")), + "expected project {project} in the provider file:\n{content}" + ); + Ok(()) +} + +/// A profile `id` whose credential the CLI discovers in [`AUTO_CREATE_ENV`]. +/// Its endpoint makes the gateway inject the credential without a policy +/// binding; nothing connects to it. +fn auto_create_profile(id: &str) -> String { + format!( + "id: {id} +display_name: E2E Auto-created Provider +category: other +credentials: + - name: api_key + env_vars: [{AUTO_CREATE_ENV}] + required: true + auth_style: header + header_name: x-api-key +discovery: + credentials: [api_key] +endpoints: + - host: api.auto-provider.example + port: 443 + protocol: rest + access: read-write + enforcement: enforce +binaries: [/usr/bin/curl] +" + ) +} + +/// Whether `value` is the placeholder for the credential in `key`, in the +/// unversioned or the revisioned form. +fn is_placeholder_for(value: &str, key: &str) -> bool { + value == format!("{PLACEHOLDER_PREFIX}{key}") + || (value.starts_with(&format!("{PLACEHOLDER_PREFIX}v")) + && value.ends_with(&format!("_{key}"))) +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index 764889a1ee..aee901d808 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -97,9 +97,6 @@ let "local_driver_token_restart" "podman_corporate_proxy" "podman_gateway_start" - # Its move to conformance is deferred until the isolation work in #4120; see - # #4246. - "provider_auto_create" # Needs the SPIFFE fixtures that the wrapper sets up. Wrapper lane. "provider_token_exchange" # One test times out reaching support-container IPs from the sandbox in the From 7d65705b9e5e9591c7f1478cafdb20eef152ba15 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:33:55 -0800 Subject: [PATCH 03/10] test(testkit): add service_get for exposed sandbox services Requests go to the gateway's endpoint with the service URL's host as the Host header, which is how the gateway routes service traffic. Over TLS the client trusts the gateway CA and presents the registered gateway's client certificate, through openshell-crypto's rustls configuration. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 3 + tests/suites/drivers/podman/Cargo.lock | 3 + tests/suites/features/Cargo.lock | 3 + tests/support/Cargo.lock | 3 + tests/testkit/Cargo.lock | 3 + tests/testkit/README.md | 4 + tests/testkit/testkit/Cargo.toml | 5 + tests/testkit/testkit/src/error.rs | 6 + tests/testkit/testkit/src/gateway.rs | 6 + tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/service.rs | 190 +++++++++++++++++++++++++ 11 files changed, 228 insertions(+) create mode 100644 tests/testkit/testkit/src/service.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index cf08ce94fe..3768b6b33c 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2405,11 +2405,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index 459d6889c4..f9615aec41 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2393,11 +2393,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 4c65ad4021..2d2bc6cf7d 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2477,11 +2477,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index bc18df94cd..65259703e7 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2411,11 +2411,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 7e808b028d..17fc9196f3 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2384,11 +2384,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index b136777dc3..f8aef833f0 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -118,6 +118,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. Chain `.config(key, value)` before the `.await` to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. +- `cx.service_get(service_url, headers)` to request an exposed sandbox + service. It connects to the gateway's endpoint and sends the service URL's + host as the Host header, so the test machine doesn't need to resolve + service hosts. It returns the status and body whatever the status. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 7b8ebd90ab..464d5b0f5b 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -17,6 +17,7 @@ name = "testkit" [dependencies] openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-policy = { path = "../../../crates/openshell-policy" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } @@ -26,6 +27,10 @@ libtest-mimic = "0.8" linkme = "0.3" quick-junit = "0.8" rand = "0.9" +# Service requests use a rustls configuration from openshell-crypto, so +# reqwest brings no crypto provider or root store of its own. +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider"] } +rustls = { version = "0.23", default-features = false, features = ["std"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 18ce6138c7..574979c1b8 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -99,6 +99,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::from_error(&error) + } +} + impl From for Error { fn from(error: std::io::Error) -> Self { Self::from_error(&error) diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index fef05833b7..79bd5fe827 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -121,6 +121,12 @@ impl Gateway { }) } + /// The directory holding the gateway's CA and client certificate, for + /// an mTLS gateway. + pub(crate) fn mtls_dir(&self) -> Option<&Path> { + self.mtls_dir.as_deref() + } + /// One line naming the gateway, for test logs. pub fn describe(&self) -> String { let auth = match (&self.mtls_dir, self.endpoint.starts_with("https://")) { diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 5772ac52fd..ea005d6a4e 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -42,6 +42,7 @@ mod provider; mod ready; mod registry; mod run; +pub mod service; mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; @@ -62,6 +63,7 @@ pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; +pub use service::{SERVICE_REQUEST_TIMEOUT, ServiceResponse}; pub use testkit_macros::test; pub use tonic::Code as GrpcCode; pub use yaml_file::YamlFile; diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs new file mode 100644 index 0000000000..f914554bf3 --- /dev/null +++ b/tests/testkit/testkit/src/service.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Requests to sandbox services that the gateway exposes. +//! +//! A service URL names the sandbox and service in its host, such as +//! `http://default--my-sandbox.openshell.localhost:8080/`. The gateway routes +//! service requests by their Host header on its own listener, so testkit +//! connects to the gateway's endpoint and sends the service URL's host as the +//! Host header. The machine that runs the tests doesn't need to resolve +//! service hosts. +//! +//! The request keeps the service URL's scheme. A gateway listening on +//! loopback reports `http` service URLs and accepts plaintext service +//! requests on its TLS listener. Over TLS, testkit verifies the gateway's +//! certificate against the endpoint's host and presents the registered +//! gateway's client certificate. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::Url; +use reqwest::header::HOST; +use rustls::RootCertStore; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +use crate::context::Context; +use crate::error::{Error, Result, ResultExt as _}; +use crate::gateway::Gateway; + +/// How long one service request may take. +pub const SERVICE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// The response to a service request. +#[derive(Debug)] +pub struct ServiceResponse { + pub status: u16, + pub body: String, +} + +impl Context { + /// Send `GET service_url` with `headers` through the test's gateway and + /// return the response, whatever its status. See [the module + /// docs](crate::service) for how the request reaches the gateway. + pub async fn service_get( + &self, + service_url: &str, + headers: &[(&str, &str)], + ) -> Result { + let cli = self.cli_handle(); + let request = ServiceRequest::new(cli.gateway(), service_url)?; + let mut builder = client(cli.gateway())? + .get(request.target) + .header(HOST, &request.host); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let what = format!("GET {service_url} through {}", cli.gateway().endpoint); + let response = builder.send().await.context(&what)?; + let status = response.status().as_u16(); + let body = response.text().await.context(&what)?; + Ok(ServiceResponse { status, body }) + } +} + +/// Where a service request goes, and the Host header that selects the +/// service. +#[derive(Debug, PartialEq, Eq)] +struct ServiceRequest { + target: Url, + host: String, +} + +impl ServiceRequest { + fn new(gateway: &Gateway, service_url: &str) -> Result { + let service = parse(service_url, "service URL")?; + let endpoint = parse(&gateway.endpoint, "gateway endpoint")?; + let service_host = service + .host_str() + .ok_or_else(|| Error::msg(format!("service URL {service_url} has no host")))?; + let host = match service.port() { + Some(port) => format!("{service_host}:{port}"), + None => service_host.to_string(), + }; + let mut target = service.clone(); + target + .set_host(endpoint.host_str()) + .map_err(Error::from_display) + .context(format!("send {service_url} to {}", gateway.endpoint))?; + target + .set_port(endpoint.port_or_known_default()) + .map_err(|()| Error::msg(format!("send {service_url} to {}", gateway.endpoint)))?; + Ok(Self { target, host }) + } +} + +fn parse(url: &str, what: &str) -> Result { + Url::parse(url) + .map_err(Error::from_display) + .context(format!("parse {what} {url}")) +} + +/// An HTTP client that trusts the gateway's CA and presents its client +/// certificate. Without a registered mTLS gateway it trusts no certificate, +/// so only plaintext requests succeed. +fn client(gateway: &Gateway) -> Result { + let builder = openshell_crypto::tls::client_builder(); + let tls = match gateway.mtls_dir() { + Some(dir) => { + let mut roots = RootCertStore::empty(); + for certificate in certificates(&dir.join("ca.crt"))? { + roots + .add(certificate) + .map_err(Error::from_display) + .context("trust the gateway CA")?; + } + builder + .with_root_certificates(roots) + .with_client_auth_cert( + certificates(&dir.join("tls.crt"))?, + private_key(&dir.join("tls.key"))?, + ) + .map_err(Error::from_display) + .context("use the gateway client certificate")? + } + None => builder + .with_root_certificates(RootCertStore::empty()) + .with_no_client_auth(), + }; + reqwest::Client::builder() + .use_preconfigured_tls(Arc::new(tls)) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(SERVICE_REQUEST_TIMEOUT) + .build() + .context("build the service HTTP client") +} + +fn certificates(path: &Path) -> Result>> { + CertificateDer::pem_file_iter(path) + .and_then(Iterator::collect) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +fn private_key(path: &Path) -> Result> { + PrivateKeyDer::from_pem_file(path) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::ServiceRequest; + use crate::gateway::Gateway; + + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { + let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); + ServiceRequest::new(&gateway, service_url).expect("service request") + } + + /// A loopback gateway with TLS reports `http` service URLs; the request + /// goes to the gateway's address in plaintext, with the service as Host. + #[test] + fn plaintext_service_on_a_tls_gateway() { + let request = request( + "https://127.0.0.1:17670", + "http://default--sb.openshell.localhost:17670/path?q=1", + ); + assert_eq!(request.target.as_str(), "http://127.0.0.1:17670/path?q=1"); + assert_eq!(request.host, "default--sb.openshell.localhost:17670"); + } + + /// The request goes to the gateway's port even when the service URL + /// names another one, such as the gateway's port inside a cluster. + #[test] + fn service_port_differs_from_the_endpoint() { + let request = request( + "https://openshell.openshell.svc.cluster.local:8443", + "https://default--sb--web.openshell.localhost/", + ); + assert_eq!( + request.target.as_str(), + "https://openshell.openshell.svc.cluster.local:8443/" + ); + assert_eq!(request.host, "default--sb--web.openshell.localhost"); + } +} From 2ab1ef4cfeb840ff3d7cd239eb05ad13aaa70688 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:37:08 -0800 Subject: [PATCH 04/10] test(features): port service bearer passthrough to a service-exposure suite Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 5 + TESTING.md | 6 + e2e/rust/Cargo.lock | 2 - e2e/rust/Cargo.toml | 7 - e2e/rust/tests/service_bearer_passthrough.rs | 350 ------------------ tests/suites/features/Cargo.lock | 8 + tests/suites/features/Cargo.toml | 2 +- .../features/service-exposure/Cargo.toml | 16 + .../tests/service_exposure.rs | 119 ++++++ 9 files changed, 155 insertions(+), 360 deletions(-) delete mode 100644 e2e/rust/tests/service_bearer_passthrough.rs create mode 100644 tests/suites/features/service-exposure/Cargo.toml create mode 100644 tests/suites/features/service-exposure/tests/service_exposure.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 86073d5b56..1bed77dced 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -105,6 +105,11 @@ jobs: {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 5a8a3942aa..91de7ca39c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -656,6 +656,12 @@ supervises its `ssh` process: it reconnects after losing the SSH transport and forwards a SIGTERM to `ssh`. The tests find those processes in `/proc`, so they need a Linux test host. CI runs them on Docker rootful. +The `service-exposure` feature suite (`tests/suites/features/service-exposure`) +exposes a service in a sandbox and checks how the gateway forwards the client's +`Authorization` header. The test sends requests to the gateway's endpoint with +the service URL's host as the Host header, so the test machine doesn't resolve +service hosts. CI runs it on the same environments as `host-gateway`. + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 287d2c65ec..38458f7de8 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2204,7 +2204,6 @@ dependencies = [ "opentelemetry-proto", "prost", "rand 0.9.5", - "rustls", "serde", "serde_json", "serial_test", @@ -2212,7 +2211,6 @@ dependencies = [ "sha2 0.10.9", "tempfile", "tokio", - "tokio-rustls", "tokio-stream", "tonic", "tonic-prost", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index a70df6d3c4..13ba00064d 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -71,11 +71,6 @@ name = "vm_overlay" path = "tests/vm_overlay.rs" required-features = ["e2e-vm"] -[[test]] -name = "service_bearer_passthrough" -path = "tests/service_bearer_passthrough.rs" -required-features = ["e2e-docker"] - [[test]] name = "rootfs_tar" path = "tests/rootfs_tar.rs" @@ -271,13 +266,11 @@ sha1 = "0.10" sha2 = "0.10" hex = "0.4" rand = "0.9" -rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tonic = { version = "0.14", features = ["transport"] } tonic-prost = "0.14" -tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tower = "0.5" url = "2" diff --git a/e2e/rust/tests/service_bearer_passthrough.rs b/e2e/rust/tests/service_bearer_passthrough.rs deleted file mode 100644 index 75415c7df2..0000000000 --- a/e2e/rust/tests/service_bearer_passthrough.rs +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! Verifies application bearer authorization behavior through an exposed -//! `OpenShell` service. - -use std::fs::File; -use std::io::BufReader; -use std::net::Ipv4Addr; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use http_body_util::{BodyExt as _, Empty}; -use hyper::client::conn::http1; -use hyper::{Request, StatusCode, header}; -use hyper_util::rt::TokioIo; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard}; -use rustls::RootCertStore; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; -use serde_json::Value; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; -use tokio::time::{sleep, timeout}; -use tokio_rustls::TlsConnector; -use url::Position; - -const SERVICE_PORT: &str = "4500"; -const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; -const READY_TIMEOUT: Duration = Duration::from_mins(1); -const HEADER_ECHO_SERVER: &str = r#" -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - body = self.headers.get("Authorization", "").encode() - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, _format, *_args): - pass - -ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() -"#; - -async fn run_cli(args: &[&str]) -> Result { - let mut command = openshell_cmd(); - command - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - command - .output() - .await - .map_err(|error| format!("failed to run openshell: {error}")) -} - -enum ServiceTransport { - Http, - Https { - connector: TlsConnector, - server_name: ServerName<'static>, - }, -} - -struct ServiceTarget { - port: u16, - authority: String, - path: String, - transport: ServiceTransport, -} - -impl ServiceTarget { - fn from_url(url: &str) -> Result { - let url = url::Url::parse(url).map_err(|error| format!("invalid service URL: {error}"))?; - let host = url - .host_str() - .ok_or_else(|| "service URL omitted its host".to_string())?; - let port = url - .port_or_known_default() - .ok_or_else(|| "service URL omitted its port".to_string())?; - let transport = match url.scheme() { - "http" => ServiceTransport::Http, - "https" => ServiceTransport::Https { - connector: e2e_tls_connector()?, - server_name: ServerName::try_from(host.to_string()) - .map_err(|error| format!("invalid service TLS server name: {error}"))?, - }, - scheme => return Err(format!("unsupported service URL scheme {scheme:?}")), - }; - let authority = url[Position::BeforeHost..Position::AfterPort].to_string(); - let path = url.query().map_or_else( - || url.path().to_string(), - |query| format!("{}?{query}", url.path()), - ); - - Ok(Self { - port, - authority, - path, - transport, - }) - } -} - -fn e2e_mtls_dir() -> Result { - let config_home = std::env::var_os("XDG_CONFIG_HOME") - .ok_or_else(|| "XDG_CONFIG_HOME is required for an HTTPS service URL".to_string())?; - let gateway = std::env::var("OPENSHELL_GATEWAY") - .map_err(|_| "OPENSHELL_GATEWAY is required for an HTTPS service URL".to_string())?; - Ok(PathBuf::from(config_home) - .join("openshell/gateways") - .join(gateway) - .join("mtls")) -} - -fn load_certificates( - path: &Path, - description: &str, -) -> Result>, String> { - let file = File::open(path) - .map_err(|error| format!("open {description} '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|error| format!("parse {description} '{}': {error}", path.display())) -} - -fn load_private_key(path: &Path) -> Result, String> { - let file = File::open(path) - .map_err(|error| format!("open client TLS key '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - PrivateKeyDer::pem_reader_iter(&mut reader) - .next() - .transpose() - .map_err(|error| format!("parse client TLS key '{}': {error}", path.display()))? - .ok_or_else(|| format!("client TLS key '{}' is empty", path.display())) -} - -fn e2e_tls_connector() -> Result { - let mtls_dir = e2e_mtls_dir()?; - let ca_path = mtls_dir.join("ca.crt"); - let mut roots = RootCertStore::empty(); - for certificate in load_certificates(&ca_path, "gateway CA certificate")? { - roots.add(certificate).map_err(|error| { - format!( - "add gateway CA certificate '{}': {error}", - ca_path.display() - ) - })?; - } - let client_certificates = - load_certificates(&mtls_dir.join("tls.crt"), "client TLS certificate")?; - let client_key = load_private_key(&mtls_dir.join("tls.key"))?; - let config = openshell_crypto::tls::client_builder() - .with_root_certificates(roots) - .with_client_auth_cert(client_certificates, client_key) - .map_err(|error| format!("build e2e mTLS client configuration: {error}"))?; - Ok(TlsConnector::from(Arc::new(config))) -} - -async fn request_over_stream( - stream: S, - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) - .await - .map_err(|error| format!("start HTTP connection: {error}"))?; - tokio::spawn(async move { - let _ = connection.await; - }); - - let request = Request::builder() - .uri(&target.path) - .header(header::HOST, &target.authority) - .header(header::AUTHORIZATION, authorization) - .body(Empty::::new()) - .map_err(|error| format!("build service request: {error}"))?; - let response = sender - .send_request(request) - .await - .map_err(|error| format!("send service request: {error}"))?; - let status = response.status(); - let body = response - .into_body() - .collect() - .await - .map_err(|error| format!("read service response: {error}"))? - .to_bytes(); - let body = String::from_utf8(body.to_vec()) - .map_err(|error| format!("service returned non-UTF-8 data: {error}"))?; - Ok((status, body)) -} - -#[allow( - clippy::disallowed_methods, - reason = "the e2e workspace does not depend on openshell-core" -)] -async fn request_service( - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> { - // The service hostname is a virtual routing authority. Dial the loopback - // gateway directly so the test does not depend on the host resolver - // recognizing arbitrary subdomains of `.localhost`. - let stream = TcpStream::connect((Ipv4Addr::LOCALHOST, target.port)) - .await - .map_err(|error| format!("connect to loopback service gateway: {error}"))?; - let _ = stream.set_nodelay(true); - match &target.transport { - ServiceTransport::Http => request_over_stream(stream, target, authorization).await, - ServiceTransport::Https { - connector, - server_name, - } => { - let stream = connector - .connect(server_name.clone(), stream) - .await - .map_err(|error| format!("start service TLS connection: {error}"))?; - request_over_stream(stream, target, authorization).await - } - } -} - -async fn wait_for_authorization(url: &str, expected: &str) -> Result<(), String> { - // Parse the URL and load TLS material before the retry loop so permanent - // test-configuration errors fail immediately instead of looking like - // service-readiness timeouts. - let target = ServiceTarget::from_url(url)?; - let mut last_observation = "no request attempted".to_string(); - let result = timeout(READY_TIMEOUT, async { - loop { - match request_service(&target, BEARER_TOKEN).await { - Ok((StatusCode::OK, body)) if body == expected => return Ok(()), - Ok((StatusCode::OK, body)) => { - return Err(format!( - "service received unexpected Authorization value: {body:?}" - )); - } - Ok((status, body)) - if matches!( - status, - StatusCode::BAD_GATEWAY - | StatusCode::PRECONDITION_FAILED - | StatusCode::SERVICE_UNAVAILABLE - ) => - { - last_observation = - format!("service returned retryable status {status} with body {body:?}"); - sleep(Duration::from_millis(250)).await; - } - Err(error) => { - last_observation = error; - sleep(Duration::from_millis(250)).await; - } - Ok((status, body)) => { - return Err(format!( - "service returned unexpected status {status} with body {body:?}" - )); - } - } - } - }) - .await; - - match result { - Ok(result) => result, - Err(_) => Err(format!( - "timed out waiting for the exposed service; last observation: {last_observation}" - )), - } -} - -#[tokio::test] -async fn service_bearer_passthrough_preserves_authorization_header() { - let sandbox_name = format!("svc-auth-{}", std::process::id()); - let create = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--from", - E2E_WORKLOAD_IMAGE, - "--expose", - SERVICE_PORT, - "--output", - "json", - "--detach", - "--no-tty", - "--", - "python3", - "-c", - HEADER_ECHO_SERVER, - ]) - .await - .expect("run sandbox create"); - assert!( - create.status.success(), - "sandbox create failed with exit {:?}: {}", - create.status.code(), - String::from_utf8_lossy(&create.stderr) - ); - let mut sandbox = SandboxGuard::manage_existing(sandbox_name.clone()); - - let created: Value = serde_json::from_slice(&create.stdout).expect("parse sandbox create JSON"); - let service_url = created - .get("service_urls") - .and_then(|urls| urls.get("")) - .and_then(Value::as_str) - .expect("unnamed service URL in create response"); - - wait_for_authorization(service_url, "") - .await - .expect("default mode should strip Authorization"); - - let expose = run_cli(&[ - "service", - "expose", - &sandbox_name, - SERVICE_PORT, - "--authorization-mode", - "bearer-passthrough", - ]) - .await - .expect("run service re-expose"); - assert!( - expose.status.success(), - "service re-expose failed with exit {:?}: {}", - expose.status.code(), - String::from_utf8_lossy(&expose.stderr) - ); - - wait_for_authorization(service_url, BEARER_TOKEN) - .await - .expect("passthrough mode should preserve Authorization"); - - sandbox.cleanup().await; -} diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 2d2bc6cf7d..a92741823a 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -3425,6 +3425,14 @@ dependencies = [ "serde", ] +[[package]] +name = "service-exposure" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde_json", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index e12a0a1a02..f605f7c85b 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "connect-ssh-supervision", "docker-updates", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "service-exposure"] diff --git a/tests/suites/features/service-exposure/Cargo.toml b/tests/suites/features/service-exposure/Cargo.toml new file mode 100644 index 0000000000..73b970a33d --- /dev/null +++ b/tests/suites/features/service-exposure/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "service-exposure" +description = "Features suite: requests reach a sandbox service the gateway exposes" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/service-exposure/tests/service_exposure.rs b/tests/suites/features/service-exposure/tests/service_exposure.rs new file mode 100644 index 0000000000..8b78e49f80 --- /dev/null +++ b/tests/suites/features/service-exposure/tests/service_exposure.rs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exposed sandbox services and the application's `Authorization` header. +//! +//! The sandbox runs a server that echoes the request's `Authorization` +//! header, and the test requests it through the gateway with +//! `cx.service_get`. + +use std::time::Duration; + +use testkit::{Context, Poll, ResultExt as _, poll_until}; + +const SERVICE_PORT: &str = "4500"; +const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; +/// How long an exposed service may take to start answering, or to apply a +/// changed authorization mode. +const SERVICE_TIMEOUT: Duration = Duration::from_mins(1); +const HEADER_ECHO_SERVER: &str = r#" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = self.headers.get("Authorization", "").encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + pass + +ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() +"#; + +/// An exposed service doesn't receive the client's `Authorization` header by +/// default. Exposing it again with `--authorization-mode bearer-passthrough` +/// forwards the header unchanged. +#[testkit::test] +async fn bearer_passthrough_preserves_authorization_header(cx: &mut Context) -> testkit::Result { + let builder = cx + .cli_sandbox("svc") + .image(testkit::python_image()) + .args(["--expose", SERVICE_PORT, "--output", "json", "--detach"]) + .command(["python3", "-c", HEADER_ECHO_SERVER]); + let name = builder.name().to_string(); + let created: serde_json::Value = builder + .run() + .await? + .success() + .context(format!("create sandbox {name}"))? + .json()?; + let sandbox = cx.adopt(&name).await?; + let service_url = created["service_urls"][""] + .as_str() + .ok_or_else(|| testkit::Error::msg(format!("no unnamed service URL in {created}")))? + .to_string(); + sandbox.wait_ready().await?; + + wait_for_authorization(cx, &service_url, "") + .await + .context("the default mode strips Authorization")?; + + cx.cli([ + "service", + "expose", + &name, + SERVICE_PORT, + "--authorization-mode", + "bearer-passthrough", + ]) + .await? + .success()?; + wait_for_authorization(cx, &service_url, BEARER_TOKEN) + .await + .context("bearer-passthrough mode preserves Authorization") +} + +/// Request `service_url` with [`BEARER_TOKEN`] until the service echoes +/// `expected`. Connection errors and the gateway's statuses for a service +/// that isn't reachable yet are retried; any other answer fails. +async fn wait_for_authorization( + cx: &Context, + service_url: &str, + expected: &str, +) -> testkit::Result { + poll_until( + &format!("{service_url} to receive Authorization {expected:?}"), + SERVICE_TIMEOUT, + Duration::from_millis(250), + async || { + let response = match cx + .service_get(service_url, &[("authorization", BEARER_TOKEN)]) + .await + { + Ok(response) => response, + Err(error) => return Ok(Poll::Pending(error.to_string())), + }; + match response.status { + 200 if response.body == expected => Ok(Poll::Ready(())), + 200 => Err(testkit::Error::msg(format!( + "the service received Authorization {:?}", + response.body + ))), + // Bad gateway, precondition failed, service unavailable. + 502 | 412 | 503 => Ok(Poll::Pending(format!( + "status {} with body {:?}", + response.status, response.body + ))), + status => Err(testkit::Error::msg(format!( + "the service answered status {status} with body {:?}", + response.body + ))), + } + }, + ) + .await +} From 2aecd4626d873c5b8e7eabfce972b027139542ce Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 05/10] fix(testkit): pass the service client's rustls config by value Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/service.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs index f914554bf3..8e2ee3102a 100644 --- a/tests/testkit/testkit/src/service.rs +++ b/tests/testkit/testkit/src/service.rs @@ -17,7 +17,6 @@ //! gateway's client certificate. use std::path::Path; -use std::sync::Arc; use std::time::Duration; use reqwest::Url; @@ -130,7 +129,7 @@ fn client(gateway: &Gateway) -> Result { .with_no_client_auth(), }; reqwest::Client::builder() - .use_preconfigured_tls(Arc::new(tls)) + .use_preconfigured_tls(tls) .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(SERVICE_REQUEST_TIMEOUT) @@ -153,9 +152,18 @@ fn private_key(path: &Path) -> Result> { #[cfg(test)] mod tests { - use super::ServiceRequest; + use super::{ServiceRequest, client}; use crate::gateway::Gateway; + /// reqwest accepts the rustls configuration only as a plain + /// `ClientConfig`; anything else fails when the client is built. + #[test] + fn client_accepts_the_tls_configuration() { + let gateway = + Gateway::from_endpoint("http://127.0.0.1:8080".to_string(), "default".to_string()); + client(&gateway).expect("service HTTP client"); + } + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); ServiceRequest::new(&gateway, service_url).expect("service request") From d3301b2439881a35c7f49a1db58aa21a854b0199 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 06/10] test(conformance): create the auto-create sandbox from the workload image Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index e67a349e3a..c5b0cdfec8 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -110,6 +110,7 @@ async fn auto_created_provider_credential_available_in_sandbox( // keeps the default policy's rules from conflicting with the provider's. let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; let sandbox_name = cx.name("ac"); + let image = testkit::python_image(); cx.track(&sandbox_name); let create = cx .cli_with_env( @@ -121,6 +122,8 @@ async fn auto_created_provider_credential_available_in_sandbox( &sandbox_name, "--no-tty", "--detach", + "--from", + &image, "--policy", policy.path()?, "--provider", From e59f8f3ed482d3d74f629b88722085628a386ede Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:21:23 -0800 Subject: [PATCH 07/10] test(testkit): add a typed provider profile builder Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 16 ++ tests/suites/drivers/podman/Cargo.lock | 16 ++ tests/support/Cargo.lock | 16 ++ tests/testkit/Cargo.lock | 17 ++ tests/testkit/README.md | 5 +- tests/testkit/testkit/Cargo.toml | 4 + tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/policy.rs | 20 +- tests/testkit/testkit/src/profile.rs | 307 +++++++++++++++++++++++++ 9 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 tests/testkit/testkit/src/profile.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 3768b6b33c..357caddb55 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2368,6 +2368,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2407,6 +2422,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index f9615aec41..e5a292df0a 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2356,6 +2356,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2395,6 +2410,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 65259703e7..e7dc4969ad 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2374,6 +2374,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2413,6 +2428,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index 17fc9196f3..4dcdb5796f 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2347,6 +2347,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2386,8 +2401,10 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "reqwest", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index f8aef833f0..f6a32d86a9 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -116,7 +116,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. `id`, and `cx.create_provider(name, type, credentials)` to create a provider, each with cleanup. `cx.install_provider(name, profile, credentials)` does both. Chain `.config(key, value)` before the `.await` - to set provider config. + to set provider config. Build the profile with `testkit::Profile` and + `testkit::Credential`, whose setters are named after the profile YAML + fields; `policy::Endpoint` configures its endpoints. `profile.to_yaml()` + renders it for these calls. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.service_get(service_url, headers)` to request an exposed sandbox service. It connects to the gateway's endpoint and sends the service URL's diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 464d5b0f5b..48cd895a9f 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -19,6 +19,7 @@ openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-policy = { path = "../../../crates/openshell-policy" } +openshell-providers = { path = "../../../crates/openshell-providers" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } clap = { version = "4", features = ["derive"] } @@ -41,6 +42,9 @@ tonic = "0.14" [target.'cfg(unix)'.dependencies] libc = "0.2" +[dev-dependencies] +prost-types = "0.14" + # A test program, run by `cargo test` and nextest through `testkit::main`'s # libtest interface, that checks the program's own subcommands. [[test]] diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index ea005d6a4e..1d1b375eef 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -37,6 +37,7 @@ mod image; mod output; pub mod policy; mod poll; +pub mod profile; pub mod program; mod provider; mod ready; @@ -58,6 +59,7 @@ pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; +pub use profile::{Credential, Profile}; pub use program::main; pub use provider::ProviderBuilder; pub use ready::check as check_ready; diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..0754651a24 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -89,6 +89,10 @@ pub trait Endpoint { fn credential_binding(&mut self, provider: &str) -> &mut Self; /// `allow_uninspected_credentials: true`. fn allow_uninspected_credentials(&mut self) -> &mut Self; + /// `enforcement: enforce`. + fn enforce(&mut self) -> &mut Self; + /// `path: `. + fn path(&mut self, path: &str) -> &mut Self; } impl Endpoint for proto::NetworkEndpoint { @@ -125,6 +129,16 @@ impl Endpoint for proto::NetworkEndpoint { self.allow_uninspected_credentials = true; self } + + fn enforce(&mut self) -> &mut Self { + self.enforcement = proto::NetworkEnforcementMode::Enforce as i32; + self + } + + fn path(&mut self, path: &str) -> &mut Self { + self.path = path.to_string(); + self + } } #[cfg(test)] @@ -155,7 +169,9 @@ mod tests { &["/usr/bin/python*"], ) .websocket(Access::ReadWrite) - .allow_uninspected_credentials(); + .allow_uninspected_credentials() + .enforce() + .path("/ws"); let yaml = format!( "{BASE}network_policies: @@ -174,6 +190,8 @@ mod tests { protocol: websocket access: read-write allow_uninspected_credentials: true + enforcement: enforce + path: /ws binaries: - path: /usr/bin/python* " diff --git a/tests/testkit/testkit/src/profile.rs b/tests/testkit/testkit/src/profile.rs new file mode 100644 index 0000000000..10217ff040 --- /dev/null +++ b/tests/testkit/testkit/src/profile.rs @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Provider profiles for tests, built in Rust and rendered as the YAML that +//! [`Context::import_profile`](crate::Context::import_profile) and +//! [`Context::install_provider`](crate::Context::install_provider) take. +//! +//! ```ignore +//! use testkit::policy::{Access, Endpoint as _}; +//! +//! let profile = testkit::Profile::new(&cx.name("auto")) +//! .credential( +//! testkit::Credential::new("api_key") +//! .env_var("API_KEY") +//! .required() +//! .auth_style("header") +//! .header_name("x-api-key"), +//! ) +//! .endpoint("api.example.test", 443, |ep| { +//! ep.rest(Access::ReadWrite).enforce(); +//! }) +//! .binaries(["/usr/bin/curl"]); +//! cx.import_profile(&profile.to_yaml()?).await?; +//! ``` +//! +//! Profile endpoints are policy endpoints, so [`policy::Endpoint`] configures +//! them. Setters are named after the YAML fields they set. +//! +//! [`policy::Endpoint`]: crate::policy::Endpoint + +use openshell_providers::{ProviderTypeProfile, profile_to_yaml}; +use openshell_sdk::raw::proto; + +use crate::error::{Error, Result, ResultExt as _}; + +pub use proto::ProviderProfileCategory as Category; + +/// A provider profile. Its display name defaults to its id, and its category +/// to `other`. +#[derive(Clone, Debug)] +#[must_use] +pub struct Profile { + profile: proto::ProviderProfile, +} + +impl Profile { + /// A profile with `id`, the provider type it defines. + pub fn new(id: &str) -> Self { + Self { + profile: proto::ProviderProfile { + id: id.to_string(), + display_name: id.to_string(), + category: Category::Other as i32, + ..proto::ProviderProfile::default() + }, + } + } + + /// The profile's `id`. + pub fn id(&self) -> &str { + &self.profile.id + } + + /// `display_name`. + pub fn display_name(mut self, name: &str) -> Self { + self.profile.display_name = name.to_string(); + self + } + + /// `category`. + pub fn category(mut self, category: Category) -> Self { + self.profile.category = category as i32; + self + } + + /// Add `credential` to `credentials`. + pub fn credential(mut self, credential: Credential) -> Self { + self.profile.credentials.push(credential.credential); + self + } + + /// `discovery: {credentials: [..]}`: the credentials the CLI looks for in + /// its environment when it creates the provider itself. + pub fn discover(mut self, credentials: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.profile + .discovery + .get_or_insert_with(proto::ProviderProfileDiscovery::default) + .credentials + .extend(credentials.into_iter().map(Into::into)); + self + } + + /// Add an endpoint for `host:port` to `endpoints`, configured by + /// `configure` with [`policy::Endpoint`](crate::policy::Endpoint). + pub fn endpoint( + mut self, + host: &str, + port: u16, + configure: impl FnOnce(&mut proto::NetworkEndpoint), + ) -> Self { + let mut endpoint = proto::NetworkEndpoint { + host: host.to_string(), + port: port.into(), + ..proto::NetworkEndpoint::default() + }; + configure(&mut endpoint); + self.profile.endpoints.push(endpoint); + self + } + + /// Add `paths` to `binaries`. + pub fn binaries(mut self, paths: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.profile.binaries.extend( + paths + .into_iter() + .map(|path| proto::NetworkBinary { path: path.into() }), + ); + self + } + + /// The profile as YAML, as `profile export` writes it. + pub fn to_yaml(&self) -> Result { + profile_to_yaml(&ProviderTypeProfile::from_proto(&self.profile)) + .map_err(Error::from_display) + .context(format!("render provider profile {}", self.profile.id)) + } +} + +/// A credential in a [`Profile`]. +#[derive(Clone, Debug)] +#[must_use] +pub struct Credential { + credential: proto::ProviderProfileCredential, +} + +impl Credential { + /// A credential called `name`. + pub fn new(name: &str) -> Self { + Self { + credential: proto::ProviderProfileCredential { + name: name.to_string(), + ..proto::ProviderProfileCredential::default() + }, + } + } + + /// Add `var` to `env_vars`. + pub fn env_var(mut self, var: &str) -> Self { + self.credential.env_vars.push(var.to_string()); + self + } + + /// `required: true`. + pub fn required(mut self) -> Self { + self.credential.required = true; + self + } + + /// `auth_style`, such as `bearer` or `header`. + pub fn auth_style(mut self, style: &str) -> Self { + self.credential.auth_style = style.to_string(); + self + } + + /// `header_name`. + pub fn header_name(mut self, name: &str) -> Self { + self.credential.header_name = name.to_string(); + self + } + + /// `refresh`, for a credential the gateway refreshes. + pub fn refresh(mut self, refresh: proto::ProviderCredentialRefresh) -> Self { + self.credential.refresh = Some(refresh); + self + } +} + +#[cfg(test)] +mod tests { + use openshell_providers::parse_profile_yaml; + + use super::*; + use crate::policy::{Access, Endpoint as _}; + + fn parsed(yaml: &str) -> ProviderTypeProfile { + parse_profile_yaml(yaml).expect("the YAML parses") + } + + /// Each setter sets the YAML field it is named after, so the gateway + /// can't tell a built profile from the YAML. + #[test] + fn setters_match_the_yaml_they_name() { + let profile = Profile::new("example") + .display_name("Example") + .category(Category::Inference) + .credential( + Credential::new("api_key") + .env_var("API_KEY") + .env_var("OTHER_KEY") + .required() + .auth_style("header") + .header_name("x-api-key"), + ) + .discover(["api_key"]) + .endpoint("api.example.test", 443, |ep| { + ep.rest(Access::ReadWrite).enforce().path("/v1/**"); + }) + .endpoint("ws.example.test", 8443, |ep| { + ep.websocket(Access::ReadOnly); + }) + .binaries(["/usr/bin/curl", "/usr/bin/python*"]); + + let expected = parsed( + "id: example +display_name: Example +category: inference +credentials: + - name: api_key + env_vars: [API_KEY, OTHER_KEY] + required: true + auth_style: header + header_name: x-api-key +discovery: + credentials: [api_key] +endpoints: + - host: api.example.test + port: 443 + path: /v1/** + protocol: rest + access: read-write + enforcement: enforce + - host: ws.example.test + port: 8443 + protocol: websocket + access: read-only +binaries: [/usr/bin/curl, /usr/bin/python*] +", + ); + let yaml = profile.to_yaml().expect("renders"); + assert_eq!(parsed(&yaml), expected, "rendered:\n{yaml}"); + } + + /// A profile with only an id has the display name and category the + /// gateway requires. + #[test] + fn defaults() { + let yaml = Profile::new("bare").to_yaml().expect("renders"); + assert_eq!( + parsed(&yaml), + parsed("id: bare\ndisplay_name: bare\ncategory: other\n"), + "rendered:\n{yaml}" + ); + } + + #[test] + fn refresh() { + let refresh = proto::ProviderCredentialRefresh { + strategy: proto::ProviderCredentialRefreshStrategy::Oauth2RefreshToken as i32, + token_url: "https://issuer.example.test/token".to_string(), + scopes: vec!["openid".to_string()], + refresh_before: Some(prost_types::Duration { + seconds: 60, + nanos: 0, + }), + material: vec![proto::ProviderCredentialRefreshMaterial { + name: "refresh_token".to_string(), + required: true, + secret: true, + ..proto::ProviderCredentialRefreshMaterial::default() + }], + ..proto::ProviderCredentialRefresh::default() + }; + let yaml = Profile::new("refreshing") + .credential(Credential::new("token").refresh(refresh)) + .to_yaml() + .expect("renders"); + assert_eq!( + parsed(&yaml), + parsed( + "id: refreshing +display_name: refreshing +category: other +credentials: + - name: token + refresh: + strategy: oauth2_refresh_token + token_url: https://issuer.example.test/token + scopes: [openid] + refresh_before: 60s + material: + - name: refresh_token + required: true + secret: true +" + ), + "rendered:\n{yaml}" + ); + } +} From c7d83f854d49479409f5d947d4bf5c666c7d4224 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 21:21:23 -0800 Subject: [PATCH 08/10] test(e2e): build provider profiles with testkit::Profile Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 42 +++--- tests/suites/features/Cargo.lock | 18 ++- .../keycloak-provider-refresh/Cargo.toml | 2 +- .../tests/provider_refresh.rs | 127 ++++++------------ 4 files changed, 79 insertions(+), 110 deletions(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index c5b0cdfec8..537bb41b78 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -4,7 +4,8 @@ //! Providers attached to a sandbox: files a provider profile serves, and //! providers that `sandbox create --provider` creates from local credentials. -use testkit::{Context, ResultExt as _, YamlFile, policy}; +use testkit::policy::{self, Access, Endpoint as _}; +use testkit::{Context, Credential, Profile, ResultExt as _, YamlFile}; /// The example profile that serves `client.toml` from the provider's config. const FILES_PROFILE: &str = @@ -96,7 +97,7 @@ async fn auto_created_provider_credential_available_in_sandbox( cx: &mut Context, ) -> testkit::Result { let provider = cx - .import_profile(&auto_create_profile(&cx.name("auto"))) + .import_profile(&auto_create_profile(&cx.name("auto")).to_yaml()?) .await?; // The CLI names the provider it creates after the profile. let cli = cx.cli_handle(); @@ -176,28 +177,21 @@ async fn expect_project(sandbox: &testkit::Sandbox, path: &str, project: &str) - /// A profile `id` whose credential the CLI discovers in [`AUTO_CREATE_ENV`]. /// Its endpoint makes the gateway inject the credential without a policy /// binding; nothing connects to it. -fn auto_create_profile(id: &str) -> String { - format!( - "id: {id} -display_name: E2E Auto-created Provider -category: other -credentials: - - name: api_key - env_vars: [{AUTO_CREATE_ENV}] - required: true - auth_style: header - header_name: x-api-key -discovery: - credentials: [api_key] -endpoints: - - host: api.auto-provider.example - port: 443 - protocol: rest - access: read-write - enforcement: enforce -binaries: [/usr/bin/curl] -" - ) +fn auto_create_profile(id: &str) -> Profile { + Profile::new(id) + .display_name("E2E Auto-created Provider") + .credential( + Credential::new("api_key") + .env_var(AUTO_CREATE_ENV) + .required() + .auth_style("header") + .header_name("x-api-key"), + ) + .discover(["api_key"]) + .endpoint("api.auto-provider.example", 443, |ep| { + ep.rest(Access::ReadWrite).enforce(); + }) + .binaries(["/usr/bin/curl"]) } /// Whether `value` is the placeholder for the credential in `key`, in the diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index a92741823a..1df4ab86d2 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1833,7 +1833,7 @@ version = "0.0.0" dependencies = [ "keycloak-support", "openshell-testkit", - "tempfile", + "prost-types", ] [[package]] @@ -2428,6 +2428,21 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "openshell-providers" +version = "0.0.0" +dependencies = [ + "glob", + "noyalib", + "openshell-core", + "openshell-policy", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "url", +] + [[package]] name = "openshell-sdk" version = "0.0.0" @@ -2479,6 +2494,7 @@ dependencies = [ "openshell-core", "openshell-crypto", "openshell-policy", + "openshell-providers", "openshell-sdk", "openshell-testkit-macros", "quick-junit", diff --git a/tests/suites/features/keycloak-provider-refresh/Cargo.toml b/tests/suites/features/keycloak-provider-refresh/Cargo.toml index e326fef5cb..1848d3cf86 100644 --- a/tests/suites/features/keycloak-provider-refresh/Cargo.toml +++ b/tests/suites/features/keycloak-provider-refresh/Cargo.toml @@ -14,4 +14,4 @@ publish = false [dependencies] keycloak-support = { path = "../../../support/keycloak" } openshell-testkit = { path = "../../../testkit/testkit" } -tempfile = "3" +prost-types = "0.14" diff --git a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs index a617c24843..3c94f284d3 100644 --- a/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs +++ b/tests/suites/features/keycloak-provider-refresh/tests/provider_refresh.rs @@ -6,10 +6,10 @@ //! Keycloak is only the provider's token issuer here; the CLI still reaches //! the gateway the usual way. -use std::io::Write as _; - use keycloak_support::KeycloakContext; -use testkit::{CliOutput, Context, ResultExt as _}; +use testkit::policy::{Access, Endpoint as _}; +use testkit::sdk::raw::proto; +use testkit::{CliOutput, Context, Credential, Profile}; const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; @@ -20,34 +20,14 @@ const CREDENTIAL_KEY: &str = "KEYCLOAK_ACCESS_TOKEN"; async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) -> testkit::Result { let kc = KeycloakContext::new(cx).await?; let provider = kc.name("kc-refresh"); - let profile_id = kc.name("kc-refresh-profile"); let grant = kc.keycloak().password_grant().await?; - let profile = write_profile(&profile_id, kc.keycloak())?; - let profile_path = profile.path().to_string_lossy().into_owned(); - - // Register each delete before its create, so cleanup also covers a create - // the gateway applied but the CLI reported as failed. Both deletes succeed - // when there is nothing to delete. - delete_on_cleanup(&kc, "profile", &profile_id); - kc.cli(["profile", "import", "--file", &profile_path]) - .await? - .success()?; - delete_on_cleanup(&kc, "provider", &provider); - kc.cli_with_env( + let profile = refresh_profile(&kc.name("kc-refresh-profile"), kc.keycloak()).to_yaml()?; + kc.install_provider( + &provider, + &profile, &[(CREDENTIAL_KEY, &grant.access_token)], - [ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--credential", - CREDENTIAL_KEY, - ], ) - .await? - .success()?; + .await?; let client_id = format!("client_id={}", kc.keycloak().client_id()); kc.cli_with_env( &[("KEYCLOAK_REFRESH_TOKEN", &grant.refresh_token)], @@ -117,64 +97,43 @@ async fn revoked_refresh_grant_requires_user_reauthorization(cx: &mut Context) - Ok(()) } -/// Delete the `kind` named `name` when the test ends, since testkit only -/// cleans up sandboxes by itself. -fn delete_on_cleanup(cx: &Context, kind: &'static str, name: &str) { - let cli = cx.cli_handle(); - let name = name.to_string(); - cx.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &name]).await?.success()?; - Ok(()) - }); -} - fn combined(output: &CliOutput) -> String { format!("{}{}", output.stdout, output.stderr) } -fn write_profile( - profile_id: &str, - keycloak: &keycloak_support::Keycloak, -) -> testkit::Result { - let issuer_token_url = keycloak.token_endpoint(); - let profile = format!( - r#"id: {profile_id} -display_name: Keycloak provider refresh test -category: other -credentials: - - name: access_token - env_vars: [{CREDENTIAL_KEY}] - required: true - auth_style: bearer - header_name: authorization - refresh: - strategy: oauth2_refresh_token - token_url: {issuer_token_url} - scopes: [openid] - refresh_before_seconds: 60 - max_lifetime_seconds: 3600 - material: - - name: client_id - required: true - - name: refresh_token - required: true - secret: true -endpoints: - - host: keycloak.test.invalid - port: 443 - protocol: rest - access: read-only - enforcement: enforce -binaries: - - /usr/bin/curl -"# - ); - let mut file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .context("create provider profile")?; - file.write_all(profile.as_bytes()) - .and_then(|()| file.flush()) - .context("write provider profile")?; - Ok(file) +/// A profile whose `access_token` the gateway refreshes from `keycloak` +/// with the OAuth refresh-token grant. +fn refresh_profile(id: &str, keycloak: &keycloak_support::Keycloak) -> Profile { + let material = |name: &str, secret: bool| proto::ProviderCredentialRefreshMaterial { + name: name.to_string(), + required: true, + secret, + ..proto::ProviderCredentialRefreshMaterial::default() + }; + let seconds = |seconds| Some(prost_types::Duration { seconds, nanos: 0 }); + Profile::new(id) + .display_name("Keycloak provider refresh test") + .credential( + Credential::new("access_token") + .env_var(CREDENTIAL_KEY) + .required() + .auth_style("bearer") + .header_name("authorization") + .refresh(proto::ProviderCredentialRefresh { + strategy: proto::ProviderCredentialRefreshStrategy::Oauth2RefreshToken as i32, + token_url: keycloak.token_endpoint(), + scopes: vec!["openid".to_string()], + refresh_before: seconds(60), + max_lifetime: seconds(3600), + material: vec![ + material("client_id", false), + material("refresh_token", true), + ], + ..proto::ProviderCredentialRefresh::default() + }), + ) + .endpoint("keycloak.test.invalid", 443, |ep| { + ep.rest(Access::ReadOnly).enforce(); + }) + .binaries(["/usr/bin/curl"]) } From b5ef8663d1a01d28c44054582871d64eb155973f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 10:31:33 -0800 Subject: [PATCH 09/10] test(conformance): run the auto-create sandbox as sleep infinity Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index 537bb41b78..dc0f689179 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -131,9 +131,8 @@ async fn auto_created_provider_credential_available_in_sandbox( &provider, "--auto-providers", "--", - "sh", - "-c", - "exec sleep infinity", + "sleep", + "infinity", ], ) .await? From d79cc8ad7ecc8c029ddbab8472515e93fa934f98 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Wed, 18 Nov 2026 16:58:26 -0800 Subject: [PATCH 10/10] test(conformance): attach the provider files provider with SandboxBuilder::providers Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index dc0f689179..36fd784be9 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -28,7 +28,7 @@ async fn provider_file_open_update_and_detach(cx: &mut Context) -> testkit::Resu let sandbox = cx .sandbox("pf") - .spec(|spec| spec.providers = vec![provider.clone()]) + .providers([&provider]) .command(["sleep", "infinity"]) .create_ready() .await?;