From 89975a46c6b566c8933885162828bb80bf4a3b27 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 1/8] docs(testkit): add the tests/support environment contract table Signed-off-by: Matthew Grossman --- tests/support/README.md | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/support/README.md diff --git a/tests/support/README.md b/tests/support/README.md new file mode 100644 index 0000000000..f2ca63d396 --- /dev/null +++ b/tests/support/README.md @@ -0,0 +1,42 @@ +# Test support crates + +Each crate here is a fixture for one concern, such as a service the +environment runs, and any suite can depend on it by path. See +[Adding a feature suite](../../TESTING.md#adding-a-feature-suite) for how an +environment and a fixture share a contract. + +## Environment contract + +These are the variables that an environment, the test runner or CI sets for +tests to read. A tmachine environment publishes its variables with +`tests/tmachine/ansible/playbooks/test-env.yaml`, and `openshell-test-runner` +passes them to the tests. To run a suite against your own deployment, set them +yourself. A fixture fails the test when a variable it needs is missing; it +never skips it. Add a row when you add a variable. + +| Variable | Set by | Read by | Meaning | +|---|---|---|---| +| `OPENSHELL_TEST_CONTAINER_ENGINE` | Every Docker and Podman base environment (`playbooks/docker.yaml`, `podman-rootful.yaml`, `podman-rootless.yaml`); `e2e:oci-image` tasks | `container-engine-support` | The command that reaches the gateway's container engine, split on whitespace, such as `docker` or `sudo -n podman`. | +| `OPENSHELL_TEST_PODMAN_USERNS` | `-userns-` (`playbooks/drivers/podman/userns.yaml`) | `podman-support` | The gateway's Podman user-namespace mode: `default`, `auto`, `keep-id` or `private`. | +| `OPENSHELL_TEST_CDI_SCENARIOS` | `-cdi` (`playbooks/cdi.yaml`) | `tests/suites/features/cdi` | The path of the document that names the CDI device kind, each scenario and the checks it expects. | +| `OPENSHELL_TEST_KEYCLOAK_ISSUER` | `-keycloak` (`playbooks/keycloak.yaml`) | `keycloak-support` | The OIDC issuer URL of the test realm, reachable from the test process. | +| `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | +| `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | +| `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | +| `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | +| `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | +| `OPENSHELL_TEST_POSTGRES_URL` | `tasks/scripts/run-postgres-tests.sh` (`mise run test:rust:postgres`), or you | `openshell-server`'s PostgreSQL tests | A disposable PostgreSQL database. Never point it at one a gateway uses. | + +Other `OPENSHELL_TEST_*` names are internal test hooks and are left out on +purpose: unit-test child-process markers, the re-entry markers in +`tests/support/environment.rs`, Nix guest-cache settings, `install.sh` test +hooks and crate test settings. + +### Addresses the environment chooses + +When a test must listen on an address the environment picked, because the +gateway or another service is configured to call it, the environment +publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it. The +test never picks the address itself. Tests that bind the same address can't +run at the same time. From d90a51955c7d97f27265e51cb2bb192f6c25d088 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 15:54:41 -0800 Subject: [PATCH 2/8] test(tmachine): add a gateway_overrides role for environment gateway settings Signed-off-by: Matthew Grossman --- TESTING.md | 29 ++++++++--- .../playbooks/drivers/podman/userns.yaml | 42 ++++----------- .../roles/cdi_fixtures/tasks/main.yaml | 24 ++------- .../gateway_overrides/defaults/main.yaml | 5 ++ .../roles/gateway_overrides/tasks/main.yaml | 51 +++++++++++++++++++ .../openshell_external_driver/tasks/main.yaml | 17 ++----- .../files/merge_gateway_toml_test.py | 22 ++++++++ .../tasks/merge-settings.yaml | 7 +-- 8 files changed, 125 insertions(+), 72 deletions(-) create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml create mode 100644 tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml diff --git a/TESTING.md b/TESTING.md index 8bb56da91e..24e5fe6f05 100644 --- a/TESTING.md +++ b/TESTING.md @@ -324,7 +324,9 @@ service. clients there from the variables. A missing variable fails the test; it never skips it. - **Documentation.** Document the variables in the fixture's module docs, so - anyone can run the suite against their own deployment by setting them. + anyone can run the suite against their own deployment by setting them, and + add a row for each to the + [environment contract table](tests/support/README.md#environment-contract). The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it with: @@ -596,11 +598,26 @@ nix run .#openshell-test-runner -- run \ #### Gateway overrides in an environment -An environment overrides gateway settings by writing the tables to change to -`/etc/openshell-test/gateway-overrides.toml` in a playbook that runs before the -installer. The file holds whole tables, such as `[openshell.gateway]` or -`[openshell.drivers.podman]`, and is not a complete configuration. Each -installer applies it to the gateway's own file: +An environment overrides gateway settings with the `gateway_overrides` role, +which takes a name and the tables to change, such as `[openshell.gateway]` or +`[openshell.drivers.podman]`: + +```yaml +- name: Push configuration to supervisors + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: config-push + gateway_overrides_content: | + [openshell.gateway] + config_delivery_mode = "push" +``` + +The role writes each name as its own marked block in +`/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's +settings. Two blocks must not declare the same table. Don't write the file +directly. The file is not a complete configuration. Each installer applies it +to the gateway's own file: - `binaries` applies it to the `/etc/openshell/gateway.toml` it generates. - `deb` and `rpm` apply it to the service user's diff --git a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml index 202270100f..dc74fe8a93 100644 --- a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml +++ b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml @@ -4,18 +4,16 @@ # Configure the Podman gateway's user-namespace mode and publish it. # # `podman_userns_mode` is default, auto, keep-id or private. Every mode except -# default writes its `[openshell.drivers.podman]` settings to -# `/etc/openshell-test/gateway-overrides.toml`, which every installer applies to the active -# gateway-overrides.toml (the openshell_gateway role for `binaries`, the -# openshell_packaged_gateway role for the packages); default removes the file. The -# Podman support crate (tests/support/podman) reads the published mode and -# runs Podman's own reference container in it. +# default adds its `[openshell.drivers.podman]` settings to the gateway +# overrides (the gateway_overrides role), which every installer applies to the +# gateway's configuration. Default mode adds nothing, so userns, uidmap and +# gidmap stay unset. The Podman support crate (tests/support/podman) reads the +# published mode and runs Podman's own reference container in it. --- - name: Podman user namespaces for tests hosts: all gather_facts: false vars: - podman_userns_driver_config: /etc/openshell-test/gateway-overrides.toml podman_userns_settings: default: "" auto: | @@ -43,34 +41,14 @@ {{ podman_userns_settings.keys() | join(', ') }}, not {{ podman_userns_mode | default('unset') }} - - name: Create the test configuration directory - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config | dirname }}" - state: directory - owner: root - group: root - mode: "0755" - - name: Write the Podman driver user-namespace settings - become: true - ansible.builtin.copy: - dest: "{{ podman_userns_driver_config }}" - content: "{{ podman_userns_settings[podman_userns_mode] }}" - owner: root - group: root - mode: "0644" + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: podman-userns + gateway_overrides_content: "{{ podman_userns_settings[podman_userns_mode] }}" when: podman_userns_mode != "default" - # The fragment is the only source of userns, uidmap and gidmap settings in - # the gateway's configuration, so default mode leaves them unset. - - name: Remove Podman driver user-namespace settings for default mode - become: true - ansible.builtin.file: - path: "{{ podman_userns_driver_config }}" - state: absent - when: podman_userns_mode == "default" - - name: Publish Podman user-namespace test variables ansible.builtin.include_tasks: ../../test-env.yaml vars: diff --git a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml index 5f0c04586c..250629d90b 100644 --- a/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/cdi_fixtures/tasks/main.yaml @@ -38,25 +38,11 @@ # The tests select the scenario device per sandbox through driver config, which # the gateway rejects unless an administrator allows it. The installers add # this to the active driver's settings. -- name: Create the test settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - owner: root - group: root - mode: "0755" - -# The gateway must accept a per-sandbox CDI device, which it does only when the -# driver allows caller driver config. Every installer merges this override into -# the gateway's own configuration (see openshell_gateway/tasks/merge-settings.yaml). - name: Allow caller driver config for the CDI tests - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: cdi + gateway_overrides_content: | [openshell.drivers.{{ environment_driver }}] allow_driver_config = true - owner: root - group: root - mode: "0644" diff --git a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml new file mode 100644 index 0000000000..73e224fb07 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +gateway_overrides_path: /etc/openshell-test/gateway-overrides.toml diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml new file mode 100644 index 0000000000..5114e90e69 --- /dev/null +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Add gateway settings to /etc/openshell-test/gateway-overrides.toml, which +# the `binaries`, `deb` and `rpm` installers apply to the gateway's +# configuration (openshell_gateway/tasks/merge-settings.yaml; k3s installers +# ignore it). +# +# Inputs: +# gateway_overrides_name names the marked block, such as `cdi`; writing +# the same name again replaces its block +# gateway_overrides_content whole TOML tables, such as +# `[openshell.drivers.podman]` +# +# Each name gets its own marked block, so writers keep each other's settings +# and may run in any order. Two blocks must not declare the same table: TOML +# rejects a table defined twice, and the installer fails on it. +# +# TODO: https://github.com/NVIDIA/OpenShell/pull/5489 reads gateway.d drop-in +# files. When it lands, write each name to its own drop-in here instead. +--- +- name: Require a gateway override name and content + ansible.builtin.assert: + that: + - gateway_overrides_name is defined + - gateway_overrides_name is match('^[a-z0-9][a-z0-9-]*$') + - gateway_overrides_content is defined + - gateway_overrides_content | trim | length > 0 + fail_msg: >- + gateway_overrides needs gateway_overrides_name (lowercase letters, + digits and dashes) and non-empty gateway_overrides_content + +- name: Create the test configuration directory + become: true + ansible.builtin.file: + path: "{{ gateway_overrides_path | dirname }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Write the {{ gateway_overrides_name }} gateway overrides + become: true + ansible.builtin.blockinfile: + path: "{{ gateway_overrides_path }}" + create: true + owner: root + group: root + mode: "0644" + marker: "# {mark} {{ gateway_overrides_name }}" + block: "{{ gateway_overrides_content }}" diff --git a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml index a31363d57d..2f3d4e3872 100644 --- a/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/openshell_external_driver/tasks/main.yaml @@ -108,18 +108,11 @@ ansible.builtin.fail: msg: The external driver socket did not appear -- name: Create the environment settings directory - become: true - ansible.builtin.file: - path: /etc/openshell-test - state: directory - mode: "0755" - - name: Point the gateway at the external driver - become: true - ansible.builtin.copy: - dest: /etc/openshell-test/gateway-overrides.toml - mode: "0644" - content: | + ansible.builtin.include_role: + name: gateway_overrides + vars: + gateway_overrides_name: external-driver + gateway_overrides_content: | [openshell.drivers.{{ openshell_external_driver_name }}] socket_path = "{{ openshell_external_driver_socket }}" diff --git a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py index ba6926169a..3f37c15178 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py +++ b/tests/tmachine/ansible/roles/openshell_gateway/files/merge_gateway_toml_test.py @@ -98,3 +98,25 @@ def test_creates_from_default_else_minimal(tmp_path): run, base = merge(other, None, '[openshell.drivers.podman]\nuserns = "auto"\n') parsed = tomllib.loads(base.read_text()) assert parsed["openshell"]["version"] == 2 + + +def test_applies_every_marked_block(tmp_path): + overrides = ( + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + "# BEGIN config-push\n[openshell.gateway]\nconfig_delivery_mode = \"push\"\n# END config-push\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode == 0, run.stderr + parsed = tomllib.loads(base.read_text()) + assert parsed["openshell"]["drivers"]["podman"]["userns"] == "auto" + assert parsed["openshell"]["gateway"]["config_delivery_mode"] == "push" + + +def test_rejects_blocks_that_declare_the_same_table(tmp_path): + overrides = ( + "# BEGIN cdi\n[openshell.drivers.podman]\nallow_driver_config = true\n# END cdi\n" + "# BEGIN podman-userns\n[openshell.drivers.podman]\nuserns = \"auto\"\n# END podman-userns\n" + ) + run, base = merge(tmp_path, BASE, overrides) + assert run.returncode != 0 + assert base.read_text() == BASE diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml index 6d167a3ec8..2b3b8e3e22 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml @@ -1,10 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# An environment overrides gateway settings by writing the tables to change, +# An environment overrides gateway settings by adding the tables to change, # such as `[openshell.drivers.podman]`, to -# /etc/openshell-test/gateway-overrides.toml before the installer runs (for -# example playbooks/drivers/podman/userns.yaml). Apply them to +# /etc/openshell-test/gateway-overrides.toml with the gateway_overrides role +# before the installer runs (for example playbooks/drivers/podman/userns.yaml). +# Apply them to # `openshell_gateway_merge_target` without replacing the file; see # files/merge-gateway-toml.py. # From a7e7bb90be75d6e9385400301ee4e76cafb4c677 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:00:13 -0800 Subject: [PATCH 3/8] test(testkit): serve TLS from host servers with a test or environment CA Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 4 + tests/suites/features/Cargo.lock | 4 + tests/support/Cargo.lock | 4 + tests/support/README.md | 2 + tests/support/host-server/Cargo.toml | 4 + tests/support/host-server/src/http.rs | 72 ++++- tests/support/host-server/src/lib.rs | 5 +- tests/support/host-server/src/tls.rs | 308 +++++++++++++++++++++ tests/support/host-server/src/websocket.rs | 3 +- 9 files changed, 391 insertions(+), 15 deletions(-) create mode 100644 tests/support/host-server/src/tls.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 0d17b24979..53c4bdee9e 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1285,9 +1285,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 3a0b5a30b2..2f966d222c 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1317,9 +1317,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 044a44aa3e..440b9ec84b 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1281,9 +1281,13 @@ dependencies = [ name = "host-server-support" version = "0.0.0" dependencies = [ + "openshell-crypto", "openshell-testkit", + "rcgen", + "rustls", "serde_json", "tokio", + "tokio-rustls", "tungstenite", ] diff --git a/tests/support/README.md b/tests/support/README.md index f2ca63d396..c13ba758e1 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -23,6 +23,8 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | +| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | +| `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | | `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | | `OPENSHELL_TEST_UNSUPPORTED` | CI, from `tests/suites/conformance/unsupported.yaml` (`tasks/scripts/conformance-unsupported.sh`), and `e2e/support/conformance.sh` for the host lanes; the runner passes it to the tests | testkit programs | Tests the environment can't pass yet, as `=` entries. See [Unsupported tests](../testkit/README.md#unsupported-tests). | diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index cf51fd6a4c..ccac94f0d1 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -12,9 +12,13 @@ repository = "https://github.com/NVIDIA/OpenShell" publish = false [dependencies] +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-testkit = { path = "../../testkit/testkit" } +rcgen = { version = "0.14.10", default-features = false, features = ["pem"] } +rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } serde_json = "1" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["tls12"] } tungstenite = "0.26" [dev-dependencies] diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index b93e6dddcc..8b1f830faa 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -7,10 +7,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; -use tokio::net::TcpStream; +use tokio::io::{AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _}; use tokio::task::JoinHandle; +use tokio_rustls::TlsAcceptor; +use crate::tls::Identity; use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; /// A request an [`HttpRecorder`] read. A chunked body arrives decoded. @@ -171,6 +172,30 @@ impl HttpRecorder { /// Answer every request with `handler`'s response. pub async fn serve( handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(None, handler).await + } + + /// Answer every request over TLS with `200` and `body`, presenting + /// `identity`'s certificate. + pub async fn start_tls(identity: &Identity, body: impl Into) -> Result { + let body = body.into(); + Self::serve_tls(identity, move |_| Response::text(body.clone())).await + } + + /// Answer every request over TLS with `handler`'s response, presenting + /// `identity`'s certificate. A connection that fails the TLS handshake + /// counts in [`connections`](Self::connections) but records no request. + pub async fn serve_tls( + identity: &Identity, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + Self::listen(Some(identity.acceptor()?), handler).await + } + + async fn listen( + tls: Option, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { let (listener, port) = bind().await?; let handler: Arc = Arc::new(handler); @@ -181,11 +206,21 @@ impl HttpRecorder { let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { task_connections.fetch_add(1, Ordering::AcqRel); - tokio::spawn(respond( - stream, - Arc::clone(&handler), - Arc::clone(&task_requests), - )); + let _ = stream.set_nodelay(true); + let local_addr = stream.local_addr().ok(); + let handler = Arc::clone(&handler); + let requests = Arc::clone(&task_requests); + let tls = tls.clone(); + tokio::spawn(async move { + match tls { + None => respond(stream, local_addr, handler, requests).await, + Some(tls) => { + if let Ok(stream) = tls.accept(stream).await { + respond(stream, local_addr, handler, requests).await; + } + } + } + }); } }); Ok(Self { @@ -239,10 +274,14 @@ impl Drop for HttpRecorder { /// Read requests, record each, and answer with `handler` until a response /// or request closes the connection. -async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc>>) { - let _ = stream.set_nodelay(true); +async fn respond( + mut stream: impl AsyncRead + AsyncWrite + Unpin, + local_addr: Option, + handler: Arc, + requests: Arc>>, +) { let mut buffer = Vec::new(); - while let Some(request) = read_request(&mut stream, &mut buffer).await { + while let Some(request) = read_request(&mut stream, local_addr, &mut buffer).await { let response = handler(&request); let close = !response.keep_alive || request @@ -260,13 +299,18 @@ async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc) -> Option { +/// `buffer`, and leave any bytes after it in `buffer`. `local_addr` is the +/// address the connection arrived on. +pub(crate) async fn read_request( + stream: &mut (impl AsyncRead + Unpin), + local_addr: Option, + buffer: &mut Vec, +) -> Option { let mut chunk = [0_u8; 4096]; loop { if let Some((mut request, length)) = parse_request(buffer) { buffer.drain(..length); - request.local_addr = stream.local_addr().ok(); + request.local_addr = local_addr; return Some(request); } match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { @@ -337,6 +381,8 @@ fn decode_chunked(bytes: &[u8]) -> Option<(Vec, usize)> { #[cfg(test)] mod tests { + use tokio::net::TcpStream; + use super::*; async fn connect(port: u16) -> TcpStream { diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index e9781bf502..374d3be419 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -11,7 +11,8 @@ //! host, such as one on a remote cluster, can't run tests that use them. //! //! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's -//! [`Response`], and records each parsed [`Request`]. +//! [`Response`], and records each parsed [`Request`]. It serves HTTPS with +//! a certificate from a [`TestCa`]. //! - [`WebSocketRecorder`] completes the WebSocket handshake and records the //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the @@ -21,6 +22,7 @@ mod http; mod tcp; +mod tls; mod websocket; use std::time::Duration; @@ -30,6 +32,7 @@ use tokio::net::TcpListener; pub use http::{HttpRecorder, Request, Response}; pub use tcp::{TcpServer, exchange}; +pub use tls::{ADDITIONAL_CA_CERT_ENV, ADDITIONAL_CA_KEY_ENV, Identity, TestCa}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; /// The name a sandbox reaches these servers by. diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs new file mode 100644 index 0000000000..6b36293b92 --- /dev/null +++ b/tests/support/host-server/src/tls.rs @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Certificate authorities for TLS host servers. +//! +//! A test signs a server certificate with a [`TestCa`] and serves it with +//! [`HttpRecorder::serve_tls`](crate::HttpRecorder::serve_tls): +//! +//! - [`TestCa::new`] makes a CA that only the test knows, so nothing trusts it +//! unless the test hands over [`TestCa::cert_pem`]. +//! - [`TestCa::from_env`] loads the environment's additional CA, which the +//! gateway is configured to trust (`additional_ca_bundle`). The environment +//! publishes the paths of its PEM certificate and key in +//! `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY`. + +use std::path::Path; +use std::sync::Arc; + +use openshell_crypto::pki::{self, KeyPair}; +use rcgen::{ + BasicConstraints, CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, + KeyUsagePurpose, +}; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use testkit::{Error, Result}; +use tokio_rustls::TlsAcceptor; + +/// The path of the environment's additional CA certificate, in PEM. +pub const ADDITIONAL_CA_CERT_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_CERT"; +/// The path of the environment's additional CA private key, in PEM. +pub const ADDITIONAL_CA_KEY_ENV: &str = "OPENSHELL_TEST_ADDITIONAL_CA_KEY"; + +/// A certificate authority that signs server certificates for host servers. +pub struct TestCa { + cert_pem: String, + issuer: Issuer<'static, KeyPair>, +} + +impl TestCa { + /// A new CA that only this test knows. + pub fn new() -> Result { + let key = pki::generate_keypair().map_err(failed("generate a test CA key"))?; + let cert = pki::self_signed(ca_params(), &key).map_err(failed("sign the test CA"))?; + Ok(Self { + cert_pem: cert.pem(), + issuer: cert.into_issuer(key), + }) + } + + /// The environment's additional CA, from the files that + /// `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY` + /// name. Fails if either is unset. + pub fn from_env() -> Result { + Self::from_vars(|name| std::env::var(name).ok()) + } + + fn from_vars(var: impl Fn(&str) -> Option) -> Result { + let path = |name: &str| { + var(name) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + Error::msg(format!( + "{name} is not set; run this test in an environment with an additional CA" + )) + }) + }; + let cert = path(ADDITIONAL_CA_CERT_ENV)?; + let key = path(ADDITIONAL_CA_KEY_ENV)?; + Self::from_files(cert.as_ref(), key.as_ref()) + } + + /// A CA from its PEM certificate and private key files. + pub fn from_files(cert: &Path, key: &Path) -> Result { + let read = |path: &Path| { + std::fs::read_to_string(path) + .map_err(|error| Error::msg(format!("read {}: {error}", path.display()))) + }; + let cert_pem = read(cert)?; + let key_pem = read(key)?; + let der = CertificateDer::from_pem_slice(cert_pem.as_bytes()).map_err(failed(format!( + "parse the CA certificate {}", + cert.display() + )))?; + let key = KeyPair::from_pem(&key_pem) + .map_err(failed(format!("parse the CA key {}", key.display())))?; + let issuer = pki::issuer_from_der(&der, key) + .map_err(failed(format!("load the CA {}", cert.display())))?; + Ok(Self { cert_pem, issuer }) + } + + /// The CA certificate, in PEM, for a client that should trust it. + pub fn cert_pem(&self) -> &str { + &self.cert_pem + } + + /// A server certificate for `names`, signed by this CA. A name that parses + /// as an IP address becomes an IP address entry. + pub fn leaf(&self, names: &[&str]) -> Result { + let names: Vec = names.iter().map(ToString::to_string).collect(); + let first = names + .first() + .cloned() + .ok_or_else(|| Error::msg("a server certificate needs at least one name"))?; + let mut params = CertificateParams::new(names.clone()) + .map_err(failed(format!("server certificate names {names:?}")))?; + params.distinguished_name.push(DnType::CommonName, first); + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let key = pki::generate_keypair().map_err(failed("generate a server key"))?; + let cert = pki::signed_by_issuer(params, &key, &self.issuer) + .map_err(failed(format!("sign a server certificate for {names:?}")))?; + let key_der = key + .serialize_der() + .map_err(failed("export the server key"))?; + let key_pem = key + .serialize_pem() + .map_err(failed("export the server key"))?; + Ok(Identity { + cert_pem: cert.pem(), + key_pem, + cert_der: cert.der().clone(), + key_der, + }) + } +} + +/// A server certificate and its private key. +#[derive(Clone)] +pub struct Identity { + pub cert_pem: String, + pub key_pem: String, + cert_der: CertificateDer<'static>, + key_der: Vec, +} + +impl Identity { + pub(crate) fn acceptor(&self) -> Result { + let key = PrivateKeyDer::Pkcs8(self.key_der.clone().into()); + let mut config = openshell_crypto::tls::try_server_builder() + .map_err(failed("build a TLS server configuration"))? + .with_no_client_auth() + .with_single_cert(vec![self.cert_der.clone()], key) + .map_err(failed("load the server certificate"))?; + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + Ok(TlsAcceptor::from(Arc::new(config))) + } +} + +impl std::fmt::Debug for Identity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Identity") + .field("cert_pem", &self.cert_pem) + .finish_non_exhaustive() + } +} + +fn ca_params() -> CertificateParams { + let mut params = CertificateParams::default(); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell test CA"); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::CrlSign, + KeyUsagePurpose::DigitalSignature, + ]; + params +} + +fn failed(context: impl std::fmt::Display) -> impl FnOnce(E) -> Error { + move |error| Error::msg(format!("{context}: {error}")) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use rustls::pki_types::ServerName; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + use tokio_rustls::client::TlsStream; + + use super::*; + use crate::{HttpRecorder, Response}; + + /// Open a TLS connection to `port` on loopback as `name`, trusting `ca`. + async fn connect(ca: &str, name: &str, port: u16) -> std::io::Result> { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.as_bytes()).unwrap()) + .unwrap(); + let config = openshell_crypto::tls::try_client_builder() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await?; + stream.set_nodelay(true)?; + TlsConnector::from(Arc::new(config)) + .connect(ServerName::try_from(name.to_string()).unwrap(), stream) + .await + } + + async fn get(stream: &mut TlsStream, path: &str) -> String { + stream + .write_all(format!("GET {path} HTTP/1.1\r\nHost: test\r\n\r\n").as_bytes()) + .await + .unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + String::from_utf8_lossy(&response).into_owned() + } + + fn write_ca(dir: &str) -> (PathBuf, PathBuf, String) { + let key = pki::generate_keypair().unwrap(); + let cert = pki::self_signed(ca_params(), &key).unwrap(); + let dir = std::env::temp_dir().join(format!("{dir}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (cert_path, key_path) = (dir.join("ca.pem"), dir.join("ca-key.pem")); + std::fs::write(&cert_path, cert.pem()).unwrap(); + std::fs::write(&key_path, key.serialize_pem().unwrap()).unwrap(); + (cert_path, key_path, cert.pem()) + } + + #[tokio::test] + async fn serve_tls_answers_a_client_that_trusts_the_ca() { + let ca = TestCa::new().unwrap(); + let identity = ca.leaf(&["localhost", "127.0.0.1"]).unwrap(); + let recorder = HttpRecorder::serve_tls(&identity, |request| { + Response::text(format!("tls {}", request.path())) + }) + .await + .unwrap(); + for name in ["localhost", "127.0.0.1"] { + let mut stream = connect(ca.cert_pem(), name, recorder.port).await.unwrap(); + let response = get(&mut stream, "/hello").await; + assert!(response.starts_with("HTTP/1.1 200 OK"), "{response}"); + assert!(response.ends_with("tls /hello"), "{response}"); + } + let requests = recorder.wait_for_requests(2).await.unwrap(); + assert_eq!(requests[0].path(), "/hello"); + } + + #[tokio::test] + async fn serve_tls_is_rejected_for_another_name_or_ca() { + let ca = TestCa::new().unwrap(); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "ok") + .await + .unwrap(); + let wrong_name = connect(ca.cert_pem(), "other.test", recorder.port).await; + assert!( + wrong_name.is_err(), + "a certificate for another name was accepted" + ); + let other = TestCa::new().unwrap(); + let wrong_ca = connect(other.cert_pem(), "localhost", recorder.port).await; + assert!( + wrong_ca.is_err(), + "a certificate from another CA was accepted" + ); + assert!(recorder.requests().is_empty()); + } + + #[tokio::test] + async fn from_vars_loads_the_environment_ca() { + let (cert, key, cert_pem) = write_ca("host-server-env-ca"); + let ca = TestCa::from_vars(|name| match name { + ADDITIONAL_CA_CERT_ENV => Some(cert.display().to_string()), + ADDITIONAL_CA_KEY_ENV => Some(key.display().to_string()), + _ => None, + }) + .unwrap(); + assert_eq!(ca.cert_pem(), cert_pem); + let recorder = HttpRecorder::start_tls(&ca.leaf(&["localhost"]).unwrap(), "env") + .await + .unwrap(); + let mut stream = connect(&cert_pem, "localhost", recorder.port) + .await + .unwrap(); + assert!(get(&mut stream, "/").await.ends_with("env")); + } + + #[test] + fn from_vars_fails_when_a_variable_is_missing() { + let error = TestCa::from_vars(|name| { + (name == ADDITIONAL_CA_CERT_ENV).then(|| "/nonexistent/ca.pem".to_string()) + }) + .err() + .unwrap(); + assert!(error.to_string().contains(ADDITIONAL_CA_KEY_ENV), "{error}"); + let error = TestCa::from_vars(|_| None).err().unwrap(); + assert!( + error.to_string().contains(ADDITIONAL_CA_CERT_ENV), + "{error}" + ); + } + + #[test] + fn leaf_needs_a_name() { + assert!(TestCa::new().unwrap().leaf(&[]).is_err()); + } +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 4f38bdb8f3..11ac2606f4 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -105,7 +105,8 @@ impl Drop for WebSocketRecorder { async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream, &mut Vec::new()).await else { + let local_addr = stream.local_addr().ok(); + let Some(request) = read_request(&mut stream, local_addr, &mut Vec::new()).await else { return; }; let Some(key) = header(&request.head, "sec-websocket-key") else { From b470b8558fb0b3c4bf9cb612cd06fbd6c65e40da Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 16:30:33 -0800 Subject: [PATCH 4/8] test(testkit): let a host HTTP server listen on a chosen port Signed-off-by: Matthew Grossman --- tests/support/README.md | 5 +- tests/support/host-server/src/http.rs | 89 ++++++++++++++++++++-- tests/support/host-server/src/lib.rs | 18 +++-- tests/support/host-server/src/tcp.rs | 2 +- tests/support/host-server/src/websocket.rs | 2 +- 5 files changed, 101 insertions(+), 15 deletions(-) diff --git a/tests/support/README.md b/tests/support/README.md index c13ba758e1..a2c0a02eaa 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -39,6 +39,7 @@ hooks and crate test settings. When a test must listen on an address the environment picked, because the gateway or another service is configured to call it, the environment -publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it. The -test never picks the address itself. Tests that bind the same address can't +publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it, for +example with `HttpRecorder::builder().port(port)` from `host-server-support`. +The test never picks the address itself. Tests that bind the same address can't run at the same time. diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index 8b1f830faa..ce7d62110e 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -173,14 +173,23 @@ impl HttpRecorder { pub async fn serve( handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - Self::listen(None, handler).await + Self::builder().serve(handler).await + } + + /// A recorder on a chosen port or with TLS, such as on a port the + /// environment published: + /// + /// ```ignore + /// let upstream = HttpRecorder::builder().port(port).tls(&identity).start("ok").await?; + /// ``` + pub fn builder() -> HttpRecorderBuilder { + HttpRecorderBuilder::default() } /// Answer every request over TLS with `200` and `body`, presenting /// `identity`'s certificate. pub async fn start_tls(identity: &Identity, body: impl Into) -> Result { - let body = body.into(); - Self::serve_tls(identity, move |_| Response::text(body.clone())).await + Self::builder().tls(identity).start(body).await } /// Answer every request over TLS with `handler`'s response, presenting @@ -190,14 +199,15 @@ impl HttpRecorder { identity: &Identity, handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - Self::listen(Some(identity.acceptor()?), handler).await + Self::builder().tls(identity).serve(handler).await } async fn listen( + port: u16, tls: Option, handler: impl Fn(&Request) -> Response + Send + Sync + 'static, ) -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(port).await?; let handler: Arc = Arc::new(handler); let requests = Arc::new(Mutex::new(Vec::new())); let connections = Arc::new(AtomicUsize::new(0)); @@ -266,6 +276,47 @@ impl HttpRecorder { } } +/// Options for an [`HttpRecorder`], from [`HttpRecorder::builder`]. +#[derive(Debug, Default)] +pub struct HttpRecorderBuilder { + port: u16, + tls: Option, +} + +impl HttpRecorderBuilder { + /// Listen on `port` instead of a random free one. Fails to start if + /// something already listens there. + #[must_use] + pub fn port(mut self, port: u16) -> Self { + self.port = port; + self + } + + /// Serve TLS, presenting `identity`'s certificate. A connection that + /// fails the TLS handshake counts in + /// [`connections`](HttpRecorder::connections) but records no request. + #[must_use] + pub fn tls(mut self, identity: &Identity) -> Self { + self.tls = Some(identity.clone()); + self + } + + /// Answer every request with `200` and `body`. + pub async fn start(self, body: impl Into) -> Result { + let body = body.into(); + self.serve(move |_| Response::text(body.clone())).await + } + + /// Answer every request with `handler`'s response. + pub async fn serve( + self, + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { + let tls = self.tls.as_ref().map(Identity::acceptor).transpose()?; + HttpRecorder::listen(self.port, tls, handler).await + } +} + impl Drop for HttpRecorder { fn drop(&mut self) { self.task.abort(); @@ -442,6 +493,34 @@ mod tests { ); } + #[tokio::test] + async fn http_recorder_listens_on_a_chosen_port() { + let port = crate::free_port().await.unwrap(); + let recorder = HttpRecorder::builder() + .port(port) + .start("fixed") + .await + .unwrap(); + assert_eq!(recorder.port, port); + let mut stream = connect(port).await; + stream + .write_all(b"GET / HTTP/1.1\r\nHost: test\r\n\r\n") + .await + .unwrap(); + assert!(read_to_end(&mut stream).await.ends_with("fixed")); + + let error = HttpRecorder::builder() + .port(port) + .start("taken") + .await + .err() + .unwrap(); + assert!( + error.to_string().contains(&format!("port {port}")), + "{error}" + ); + } + #[tokio::test] async fn http_recorder_keeps_a_connection_alive_until_the_request_closes_it() { let recorder = diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 374d3be419..b369cfac0c 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -12,7 +12,8 @@ //! //! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's //! [`Response`], and records each parsed [`Request`]. It serves HTTPS with -//! a certificate from a [`TestCa`]. +//! a certificate from a [`TestCa`], and listens on a random free port unless +//! [`HttpRecorder::builder`] names one. //! - [`WebSocketRecorder`] completes the WebSocket handshake and records the //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the @@ -30,7 +31,7 @@ use std::time::Duration; use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; -pub use http::{HttpRecorder, Request, Response}; +pub use http::{HttpRecorder, HttpRecorderBuilder, Request, Response}; pub use tcp::{TcpServer, exchange}; pub use tls::{ADDITIONAL_CA_CERT_ENV, ADDITIONAL_CA_KEY_ENV, Identity, TestCa}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; @@ -43,10 +44,15 @@ const WAIT_INTERVAL: Duration = Duration::from_millis(100); const READ_TIMEOUT: Duration = Duration::from_secs(10); const MAX_REQUEST: usize = 1024 * 1024; -async fn bind() -> Result<(TcpListener, u16)> { - let listener = TcpListener::bind(("0.0.0.0", 0)) +/// Listen on every host address at `port`, or at a random free port for 0. +async fn bind(port: u16) -> Result<(TcpListener, u16)> { + let listener = TcpListener::bind(("0.0.0.0", port)) .await - .context("bind a host server")?; + .context(if port == 0 { + "bind a host server".to_string() + } else { + format!("bind a host server on port {port}") + })?; let port = listener .local_addr() .context("read the host server's address")? @@ -56,7 +62,7 @@ async fn bind() -> Result<(TcpListener, u16)> { /// A port that nothing on the test host listened on when this returned. pub async fn free_port() -> Result { - let (_listener, port) = bind().await?; + let (_listener, port) = bind(0).await?; Ok(port) } diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs index 1b6d566895..f0a93ef006 100644 --- a/tests/support/host-server/src/tcp.rs +++ b/tests/support/host-server/src/tcp.rs @@ -45,7 +45,7 @@ impl TcpServer { } async fn start(mode: Mode) -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(0).await?; let received = Arc::new(Mutex::new(Vec::new())); let connections = Arc::new(AtomicUsize::new(0)); let task_received = Arc::clone(&received); diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 11ac2606f4..5488006e83 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -34,7 +34,7 @@ struct WebSocketState { impl WebSocketRecorder { pub async fn start() -> Result { - let (listener, port) = bind().await?; + let (listener, port) = bind(0).await?; let state = Arc::new(WebSocketState::default()); let task_state = Arc::clone(&state); let task = tokio::spawn(async move { From 41184f2d6995900dc1536d6a2f92e166b19d9e01 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:45:34 -0800 Subject: [PATCH 5/8] test(tmachine): remove the Podman userns overrides block in default mode Signed-off-by: Matthew Grossman --- TESTING.md | 3 +- .../playbooks/drivers/podman/userns.yaml | 4 +-- .../gateway_overrides/defaults/main.yaml | 1 + .../roles/gateway_overrides/tasks/main.yaml | 30 +++++++++++++++++-- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/TESTING.md b/TESTING.md index 24e5fe6f05..6abd98c033 100644 --- a/TESTING.md +++ b/TESTING.md @@ -615,7 +615,8 @@ which takes a name and the tables to change, such as `[openshell.gateway]` or The role writes each name as its own marked block in `/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's -settings. Two blocks must not declare the same table. Don't write the file +settings, and `gateway_overrides_state: absent` removes a name's block. +Two blocks must not declare the same table. Don't write the file directly. The file is not a complete configuration. Each installer applies it to the gateway's own file: diff --git a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml index dc74fe8a93..feb3b3c323 100644 --- a/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml +++ b/tests/tmachine/ansible/playbooks/drivers/podman/userns.yaml @@ -6,7 +6,7 @@ # `podman_userns_mode` is default, auto, keep-id or private. Every mode except # default adds its `[openshell.drivers.podman]` settings to the gateway # overrides (the gateway_overrides role), which every installer applies to the -# gateway's configuration. Default mode adds nothing, so userns, uidmap and +# gateway's configuration. Default mode removes them, so userns, uidmap and # gidmap stay unset. The Podman support crate (tests/support/podman) reads the # published mode and runs Podman's own reference container in it. --- @@ -47,7 +47,7 @@ vars: gateway_overrides_name: podman-userns gateway_overrides_content: "{{ podman_userns_settings[podman_userns_mode] }}" - when: podman_userns_mode != "default" + gateway_overrides_state: "{{ 'absent' if podman_userns_mode == 'default' else 'present' }}" - name: Publish Podman user-namespace test variables ansible.builtin.include_tasks: ../../test-env.yaml diff --git a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml index 73e224fb07..8e9f90608e 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/defaults/main.yaml @@ -3,3 +3,4 @@ --- gateway_overrides_path: /etc/openshell-test/gateway-overrides.toml +gateway_overrides_state: present diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml index 5114e90e69..f4d73a98f8 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -11,6 +11,8 @@ # the same name again replaces its block # gateway_overrides_content whole TOML tables, such as # `[openshell.drivers.podman]` +# gateway_overrides_state `absent` removes the name's block and keeps the +# others; the default is `present` # # Each name gets its own marked block, so writers keep each other's settings # and may run in any order. Two blocks must not declare the same table: TOML @@ -24,14 +26,19 @@ that: - gateway_overrides_name is defined - gateway_overrides_name is match('^[a-z0-9][a-z0-9-]*$') - - gateway_overrides_content is defined - - gateway_overrides_content | trim | length > 0 + - gateway_overrides_state in ['present', 'absent'] + - >- + gateway_overrides_state == 'absent' or + (gateway_overrides_content is defined and + gateway_overrides_content | trim | length > 0) fail_msg: >- gateway_overrides needs gateway_overrides_name (lowercase letters, - digits and dashes) and non-empty gateway_overrides_content + digits and dashes), gateway_overrides_state present or absent, and + non-empty gateway_overrides_content unless the state is absent - name: Create the test configuration directory become: true + when: gateway_overrides_state == 'present' ansible.builtin.file: path: "{{ gateway_overrides_path | dirname }}" state: directory @@ -49,3 +56,20 @@ mode: "0644" marker: "# {mark} {{ gateway_overrides_name }}" block: "{{ gateway_overrides_content }}" + when: gateway_overrides_state == 'present' + +- name: Check for the gateway overrides file + ansible.builtin.stat: + path: "{{ gateway_overrides_path }}" + register: gateway_overrides_file + when: gateway_overrides_state == 'absent' + +- name: Remove the {{ gateway_overrides_name }} gateway overrides + become: true + ansible.builtin.blockinfile: + path: "{{ gateway_overrides_path }}" + marker: "# {mark} {{ gateway_overrides_name }}" + state: absent + when: + - gateway_overrides_state == 'absent' + - gateway_overrides_file.stat.exists From d8aa49b0b6cbd56a60558c404b2eeda1de5d65e4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:45:34 -0800 Subject: [PATCH 6/8] test(testkit): reject a mismatched additional CA key at load time Signed-off-by: Matthew Grossman --- tests/support/README.md | 6 ++++-- tests/support/host-server/src/tls.rs | 28 +++++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/support/README.md b/tests/support/README.md index a2c0a02eaa..44cf957511 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -12,7 +12,9 @@ tests to read. A tmachine environment publishes its variables with `tests/tmachine/ansible/playbooks/test-env.yaml`, and `openshell-test-runner` passes them to the tests. To run a suite against your own deployment, set them yourself. A fixture fails the test when a variable it needs is missing; it -never skips it. Add a row when you add a variable. +never skips it. Add a row when you add a variable. The runner also sets +`OPENSHELL_GATEWAY` and `OPENSHELL_BIN`, which testkit reads; see +[Running against a gateway](../testkit/README.md#running-against-a-gateway). | Variable | Set by | Read by | Meaning | |---|---|---|---| @@ -23,7 +25,7 @@ never skips it. Add a row when you add a variable. | `OPENSHELL_TEST_KEYCLOAK_CLIENT_ID` | `-keycloak` | `keycloak-support` | A client in that realm that allows password grants. | | `OPENSHELL_TEST_KEYCLOAK_USERNAME` | `-keycloak` | `keycloak-support` | The realm user that tests sign in as. | | `OPENSHELL_TEST_KEYCLOAK_PASSWORD` | `-keycloak` | `keycloak-support` | That user's password. | -| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which configures the gateway's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the gateway trusts. | +| `OPENSHELL_TEST_ADDITIONAL_CA_CERT` | The additional-CA environment, which sets the driver's `additional_ca_bundle` (added by #5729) | `host-server-support` (`TestCa::from_env`) | The path of the CA certificate, in PEM, that the supervisor trusts for sandbox egress. | | `OPENSHELL_TEST_ADDITIONAL_CA_KEY` | The additional-CA environment | `host-server-support` (`TestCa::from_env`) | The path of that CA's private key, in PEM, readable by the test user. Tests sign server certificates with it. | | `OPENSHELL_TEST_BASE_IMAGE` | Optional; you, for example to use a mirror | `testkit::base_image()` | The default sandbox image. Unset, it is `nvcr.io/nvidia/base/ubuntu:24.04`. | | `OPENSHELL_TEST_PYTHON_IMAGE` | Optional; `e2e/rust/e2e-kubernetes.sh` and `e2e-vm.sh` set a pullable image | `testkit::python_image()` | The workload image for tests that run `python3`, curl, git or openssl. Unset, it is `openshell/e2e-python:dev`, which the tmachine installers load. | diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs index 6b36293b92..a379b35cc3 100644 --- a/tests/support/host-server/src/tls.rs +++ b/tests/support/host-server/src/tls.rs @@ -9,7 +9,7 @@ //! - [`TestCa::new`] makes a CA that only the test knows, so nothing trusts it //! unless the test hands over [`TestCa::cert_pem`]. //! - [`TestCa::from_env`] loads the environment's additional CA, which the -//! gateway is configured to trust (`additional_ca_bundle`). The environment +//! supervisor trusts for sandbox egress (`additional_ca_bundle`). The environment //! publishes the paths of its PEM certificate and key in //! `OPENSHELL_TEST_ADDITIONAL_CA_CERT` and `OPENSHELL_TEST_ADDITIONAL_CA_KEY`. @@ -82,9 +82,23 @@ impl TestCa { "parse the CA certificate {}", cert.display() )))?; - let key = KeyPair::from_pem(&key_pem) + let key_pair = KeyPair::from_pem(&key_pem) .map_err(failed(format!("parse the CA key {}", key.display())))?; - let issuer = pki::issuer_from_der(&der, key) + // issuer_from_der reads the certificate's metadata only, so check the + // key here rather than at the first client handshake. + let key_der = key_pair + .serialize_der() + .map_err(failed(format!("export the CA key {}", key.display())))?; + openshell_crypto::tls::try_server_builder() + .map_err(failed("build a TLS server configuration"))? + .with_no_client_auth() + .with_single_cert(vec![der.clone()], PrivateKeyDer::Pkcs8(key_der.into())) + .map_err(failed(format!( + "the CA key {} does not match the CA certificate {}", + key.display(), + cert.display() + )))?; + let issuer = pki::issuer_from_der(&der, key_pair) .map_err(failed(format!("load the CA {}", cert.display())))?; Ok(Self { cert_pem, issuer }) } @@ -301,6 +315,14 @@ mod tests { ); } + #[test] + fn from_files_rejects_a_key_from_another_ca() { + let (cert, _, _) = write_ca("host-server-mismatch-a"); + let (_, other_key, _) = write_ca("host-server-mismatch-b"); + let error = TestCa::from_files(&cert, &other_key).err().unwrap(); + assert!(error.to_string().contains("does not match"), "{error}"); + } + #[test] fn leaf_needs_a_name() { assert!(TestCa::new().unwrap().leaf(&[]).is_err()); From 8b1fcef6e628ab01d8e97829ff7398a4a16832c4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:48:58 -0800 Subject: [PATCH 7/8] test(tmachine): require a table header in gateway overrides and skip an empty file Signed-off-by: Matthew Grossman --- .../tmachine/ansible/roles/gateway_overrides/tasks/main.yaml | 5 +++-- .../roles/openshell_gateway/tasks/merge-settings.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml index f4d73a98f8..5da35a936f 100644 --- a/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml +++ b/tests/tmachine/ansible/roles/gateway_overrides/tasks/main.yaml @@ -30,11 +30,12 @@ - >- gateway_overrides_state == 'absent' or (gateway_overrides_content is defined and - gateway_overrides_content | trim | length > 0) + gateway_overrides_content | trim is match('\\[')) fail_msg: >- gateway_overrides needs gateway_overrides_name (lowercase letters, digits and dashes), gateway_overrides_state present or absent, and - non-empty gateway_overrides_content unless the state is absent + gateway_overrides_content that starts with a table header, such as + `[openshell.gateway]`, unless the state is absent - name: Create the test configuration directory become: true diff --git a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml index 2b3b8e3e22..05a679b2f5 100644 --- a/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml +++ b/tests/tmachine/ansible/roles/openshell_gateway/tasks/merge-settings.yaml @@ -28,4 +28,4 @@ {{ '--default ' ~ openshell_gateway_merge_default if openshell_gateway_merge_default | default('') else '' }} executable: /usr/bin/python3 - when: openshell_gateway_overrides.stat.exists + when: openshell_gateway_overrides.stat.exists and openshell_gateway_overrides.stat.size > 0 From 8005863c691ac996ed118934041bca7508b48ff5 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Thu, 19 Nov 2026 21:48:58 -0800 Subject: [PATCH 8/8] test(testkit): harden host server TLS and name environment ports _PORT Add the authority key identifier to server certificates, which strict verifiers such as Python 3.13 require, time out TLS handshakes, report CA load failures as such, and keep test CA files in temporary directories. The contract table now names environment-chosen ports OPENSHELL_TEST__PORT, matching HttpRecorder's port option. Signed-off-by: Matthew Grossman --- tests/support/Cargo.lock | 1 + tests/support/README.md | 14 +++++------ tests/support/host-server/Cargo.toml | 1 + tests/support/host-server/src/http.rs | 3 ++- tests/support/host-server/src/tls.rs | 36 ++++++++++++++++++++------- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 440b9ec84b..7eb55a6e94 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1286,6 +1286,7 @@ dependencies = [ "rcgen", "rustls", "serde_json", + "tempfile", "tokio", "tokio-rustls", "tungstenite", diff --git a/tests/support/README.md b/tests/support/README.md index 44cf957511..80402e79b9 100644 --- a/tests/support/README.md +++ b/tests/support/README.md @@ -37,11 +37,11 @@ purpose: unit-test child-process markers, the re-entry markers in `tests/support/environment.rs`, Nix guest-cache settings, `install.sh` test hooks and crate test settings. -### Addresses the environment chooses +### Ports the environment chooses -When a test must listen on an address the environment picked, because the -gateway or another service is configured to call it, the environment -publishes it as `OPENSHELL_TEST__LISTEN` and the test binds it, for -example with `HttpRecorder::builder().port(port)` from `host-server-support`. -The test never picks the address itself. Tests that bind the same address can't -run at the same time. +When a test must listen on a port the environment picked, because the gateway +or another service is configured to call it, the environment publishes it as +`OPENSHELL_TEST__PORT` and the test binds that port on every host +address, for example with `HttpRecorder::builder().port(port)` from +`host-server-support`. The test never picks the port itself. Tests that bind +the same port can't run at the same time. diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index ccac94f0d1..0b91888a99 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -22,4 +22,5 @@ tokio-rustls = { version = "0.26", default-features = false, features = ["tls12" tungstenite = "0.26" [dev-dependencies] +tempfile = "3" tokio = { version = "1.43", features = ["macros", "rt"] } diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index ce7d62110e..c8804deab7 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -225,7 +225,8 @@ impl HttpRecorder { match tls { None => respond(stream, local_addr, handler, requests).await, Some(tls) => { - if let Ok(stream) = tls.accept(stream).await { + let handshake = tokio::time::timeout(READ_TIMEOUT, tls.accept(stream)); + if let Ok(Ok(stream)) = handshake.await { respond(stream, local_addr, handler, requests).await; } } diff --git a/tests/support/host-server/src/tls.rs b/tests/support/host-server/src/tls.rs index a379b35cc3..1102c306ae 100644 --- a/tests/support/host-server/src/tls.rs +++ b/tests/support/host-server/src/tls.rs @@ -94,7 +94,7 @@ impl TestCa { .with_no_client_auth() .with_single_cert(vec![der.clone()], PrivateKeyDer::Pkcs8(key_der.into())) .map_err(failed(format!( - "the CA key {} does not match the CA certificate {}", + "load the CA key {} with the CA certificate {}", key.display(), cert.display() )))?; @@ -121,6 +121,8 @@ impl TestCa { params.distinguished_name.push(DnType::CommonName, first); params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + // Strict verifiers, such as Python 3.13 and later, require it. + params.use_authority_key_identifier_extension = true; let key = pki::generate_keypair().map_err(failed("generate a server key"))?; let cert = pki::signed_by_issuer(params, &key, &self.issuer) .map_err(failed(format!("sign a server certificate for {names:?}")))?; @@ -192,6 +194,7 @@ mod tests { use std::path::PathBuf; use rustls::pki_types::ServerName; + use tempfile::TempDir; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::TcpStream; use tokio_rustls::TlsConnector; @@ -231,12 +234,10 @@ mod tests { String::from_utf8_lossy(&response).into_owned() } - fn write_ca(dir: &str) -> (PathBuf, PathBuf, String) { + fn write_ca(dir: &TempDir) -> (PathBuf, PathBuf, String) { let key = pki::generate_keypair().unwrap(); let cert = pki::self_signed(ca_params(), &key).unwrap(); - let dir = std::env::temp_dir().join(format!("{dir}-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let (cert_path, key_path) = (dir.join("ca.pem"), dir.join("ca-key.pem")); + let (cert_path, key_path) = (dir.path().join("ca.pem"), dir.path().join("ca-key.pem")); std::fs::write(&cert_path, cert.pem()).unwrap(); std::fs::write(&key_path, key.serialize_pem().unwrap()).unwrap(); (cert_path, key_path, cert.pem()) @@ -283,7 +284,8 @@ mod tests { #[tokio::test] async fn from_vars_loads_the_environment_ca() { - let (cert, key, cert_pem) = write_ca("host-server-env-ca"); + let dir = TempDir::new().unwrap(); + let (cert, key, cert_pem) = write_ca(&dir); let ca = TestCa::from_vars(|name| match name { ADDITIONAL_CA_CERT_ENV => Some(cert.display().to_string()), ADDITIONAL_CA_KEY_ENV => Some(key.display().to_string()), @@ -317,10 +319,26 @@ mod tests { #[test] fn from_files_rejects_a_key_from_another_ca() { - let (cert, _, _) = write_ca("host-server-mismatch-a"); - let (_, other_key, _) = write_ca("host-server-mismatch-b"); + let (a, b) = (TempDir::new().unwrap(), TempDir::new().unwrap()); + let (cert, _, _) = write_ca(&a); + let (_, other_key, _) = write_ca(&b); let error = TestCa::from_files(&cert, &other_key).err().unwrap(); - assert!(error.to_string().contains("does not match"), "{error}"); + assert!( + error.to_string().contains("with the CA certificate"), + "{error}" + ); + } + + #[test] + fn leaf_names_its_issuer_key() { + const AUTHORITY_KEY_IDENTIFIER: &[u8] = &[0x06, 0x03, 0x55, 0x1d, 0x23]; + let leaf = TestCa::new().unwrap().leaf(&["localhost"]).unwrap(); + assert!( + leaf.cert_der + .windows(AUTHORITY_KEY_IDENTIFIER.len()) + .any(|window| window == AUTHORITY_KEY_IDENTIFIER), + "the server certificate has no authority key identifier" + ); } #[test]