Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
74 commits
Select commit Hold shift + click to select a range
3e45721
test(testkit): let create_provider set provider config
matthewgrossman Nov 17, 2026
1440a1d
test(conformance): port provider files and auto-create tests to testkit
matthewgrossman Nov 17, 2026
7d65705
test(testkit): add service_get for exposed sandbox services
matthewgrossman Nov 17, 2026
2ab1ef4
test(features): port service bearer passthrough to a service-exposure…
matthewgrossman Nov 17, 2026
2aecd46
fix(testkit): pass the service client's rustls config by value
matthewgrossman Nov 17, 2026
d3301b2
test(conformance): create the auto-create sandbox from the workload i…
matthewgrossman Nov 17, 2026
e59f8f3
test(testkit): add a typed provider profile builder
matthewgrossman Nov 18, 2026
c7d83f8
test(e2e): build provider profiles with testkit::Profile
matthewgrossman Nov 18, 2026
b5ef866
test(conformance): run the auto-create sandbox as sleep infinity
matthewgrossman Nov 18, 2026
8d5101b
Merge remote-tracking branch 'origin/main' into test/5656-port-provid…
matthewgrossman Nov 19, 2026
d79cc8a
test(conformance): attach the provider files provider with SandboxBui…
matthewgrossman Nov 19, 2026
edbaf95
test(testkit): add compute_driver, driver_config, sandbox logs, and p…
matthewgrossman Nov 17, 2026
af21f8d
test(support): add handler responses, chunked bodies, and raw TCP ser…
matthewgrossman Nov 17, 2026
7ceb1b4
test(testkit): add policy::base_with for YAML policy sections
matthewgrossman Nov 17, 2026
fe33f62
test(conformance): port network bypass, listener, and loopback tests
matthewgrossman Nov 17, 2026
7c76808
test(conformance): port REST, GraphQL, JSON-RPC, and MCP L7 policy tests
matthewgrossman Nov 17, 2026
9b307ba
test(conformance): port transparent egress pipeline tests
matthewgrossman Nov 17, 2026
e8f47da
test(conformance): port credential header, body, and WebSocket text r…
matthewgrossman Nov 17, 2026
c5f14c1
test(conformance): port static provider credential endpoint binding test
matthewgrossman Nov 17, 2026
d6069f8
test(conformance): port live policy update tests
matthewgrossman Nov 17, 2026
b2cdd6d
test(support): add a loopback TCP exchange client to host-server
matthewgrossman Nov 17, 2026
326a806
test(conformance): port the port forward test
matthewgrossman Nov 17, 2026
a2164e8
test(features): port the IP literal egress test to routable-host-ip
matthewgrossman Nov 17, 2026
98442b2
test(e2e): remove legacy network, L7, policy update, and port forward…
matthewgrossman Nov 17, 2026
e43de85
docs(testing): describe the network, policy update, and port forward …
matthewgrossman Nov 17, 2026
8ebd4f7
test(conformance): keep sandbox names within the gateway's 19 charact…
matthewgrossman Nov 18, 2026
9c2f3d8
test(features): use the shared http probe in routable-host-ip
matthewgrossman Nov 18, 2026
083c9a6
test(conformance): use the shared sandbox clients in the network suites
matthewgrossman Nov 18, 2026
0c73a5b
docs(testkit): point the sandbox suffix note at MAX_SANDBOX_NAME_LEN
matthewgrossman Nov 18, 2026
0bd5593
test(conformance): cover the REST route and unlisted JSON-RPC methods
matthewgrossman Nov 18, 2026
644925f
test(conformance): assert documented error codes on L7 and pipelined …
matthewgrossman Nov 18, 2026
319e28e
test(conformance): require the fail-closed middleware's 403 and match…
matthewgrossman Nov 18, 2026
881cee8
test(conformance): skip sandbox log checks on multi-replica gateways
matthewgrossman Nov 19, 2026
b7b2864
test(support): record the address each host server request arrived at
matthewgrossman Nov 19, 2026
e6adae8
test(features): find the host address from the host server in routabl…
matthewgrossman Nov 19, 2026
3bc5d69
test(conformance): skip the sandbox log tests entirely on multi-repli…
matthewgrossman Nov 19, 2026
86a2569
test(conformance): mark the sandbox log tests unsupported on k3s-ha
matthewgrossman Nov 19, 2026
9959030
docs(testkit): add the tests/support environment contract table
matthewgrossman Nov 19, 2026
174cf49
test(tmachine): add a gateway_overrides role for environment gateway …
matthewgrossman Nov 19, 2026
2eb7ee9
test(testkit): serve TLS from host servers with a test or environment CA
matthewgrossman Nov 20, 2026
52ccd76
Merge remote-tracking branch 'origin/test/5728-share-environment-help…
matthewgrossman Nov 20, 2026
387c93e
Merge remote-tracking branch 'origin/main' into test/5656-port-provid…
matthewgrossman Nov 20, 2026
ad1c5d3
Merge remote-tracking branch 'origin/test/5656-port-provider-tests/ma…
matthewgrossman Nov 20, 2026
bd9f625
feat(testkit): add workspaces, token clients, and per-CLI environment
matthewgrossman Nov 20, 2026
e80b461
feat(testkit): let a CLI change its environment and workspace
matthewgrossman Nov 20, 2026
0b2e450
feat(keycloak-support): add user sessions and the browser login
matthewgrossman Nov 20, 2026
3ad35ac
test(tmachine): add an OIDC gateway environment
matthewgrossman Nov 20, 2026
2a07cbf
docs(keycloak-support): note where the extra users come from
matthewgrossman Nov 20, 2026
5868014
feat(testkit): track a workspace's sandboxes, providers and profiles
matthewgrossman Nov 20, 2026
af021e6
test(oidc-auth): port the OIDC authorization tests to testkit
matthewgrossman Nov 20, 2026
ab47b4c
test(e2e): drop the OIDC tests the oidc-auth suite now covers
matthewgrossman Nov 20, 2026
c29be49
docs(testing): describe the OIDC feature environment
matthewgrossman Nov 20, 2026
c67e467
test(features): use the host's outbound address when routable-host-ip…
matthewgrossman Nov 20, 2026
f468936
ci(e2e): run service-exposure only where gateway service routing is on
matthewgrossman Nov 20, 2026
89975a4
docs(testkit): add the tests/support environment contract table
matthewgrossman Nov 19, 2026
d90a519
test(tmachine): add a gateway_overrides role for environment gateway …
matthewgrossman Nov 19, 2026
a7e7bb9
test(testkit): serve TLS from host servers with a test or environment CA
matthewgrossman Nov 20, 2026
b470b85
test(testkit): let a host HTTP server listen on a chosen port
matthewgrossman Nov 20, 2026
41184f2
test(tmachine): remove the Podman userns overrides block in default mode
matthewgrossman Nov 20, 2026
d8aa49b
test(testkit): reject a mismatched additional CA key at load time
matthewgrossman Nov 20, 2026
8b1fcef
test(tmachine): require a table header in gateway overrides and skip …
matthewgrossman Nov 20, 2026
8005863
test(testkit): harden host server TLS and name environment ports _PORT
matthewgrossman Nov 20, 2026
e780266
fix(tmachine): give the OIDC gateway explicit TLS paths for config pr…
matthewgrossman Nov 20, 2026
9694800
Merge remote-tracking branch 'origin/test/5728-share-environment-help…
matthewgrossman Nov 20, 2026
088719a
Merge remote-tracking branch 'origin/test/5656-port-provider-tests/ma…
matthewgrossman Nov 20, 2026
8b9ac69
Merge remote-tracking branch 'origin/main' into test/5731-port-oidc-a…
matthewgrossman Nov 20, 2026
6740fa3
fix(oidc-auth): recognize the operator by its operator role
matthewgrossman Nov 20, 2026
82b5c06
fix(oidc-auth): create providers against the workspace's imported pro…
matthewgrossman Nov 20, 2026
a22f5cf
Merge remote-tracking branch 'origin/main' into test/5655-port-networ…
matthewgrossman Nov 20, 2026
891af21
Merge remote-tracking branch 'origin/main' into test/5731-port-oidc-a…
matthewgrossman Nov 20, 2026
c3ec078
test(continuity): add prost-types to the lockfile for testkit
matthewgrossman Nov 20, 2026
c6149d3
Merge remote-tracking branch 'origin/test/5655-port-network-l7-policy…
matthewgrossman Nov 20, 2026
4a0f277
Merge remote-tracking branch 'origin/test/5728-share-environment-help…
matthewgrossman Nov 20, 2026
5a9993e
chore(continuity): refresh the lockfile for testkit's new dependencies
matthewgrossman Nov 20, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ jobs:
{"environment":"fedora-podman-rootful-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-cdi","installer":"rpm","family":"features","package":"cdi","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful-oidc","installer":"deb","family":"features","package":"oidc-auth","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
Expand All @@ -111,7 +112,14 @@ jobs:
{"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
{"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"},
Expand Down
56 changes: 50 additions & 6 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,9 @@ service.
clients there from the variables. A missing variable fails the test; it
never skips it.
- **Documentation.** Document the variables in the fixture's module docs, so
anyone can run the suite against their own deployment by setting them.
anyone can run the suite against their own deployment by setting them, and
add a row for each to the
[environment contract table](tests/support/README.md#environment-contract).

The Keycloak fixture is `tests/support/keycloak`. Run the suite that uses it
with:
Expand All @@ -335,6 +337,15 @@ nix run .#openshell-test-runner -- run \
--family features --package keycloak-provider-refresh
```

The `oidc-auth` suite runs in `ubuntu-docker-rootful-oidc`, which adds a gateway
that trusts the Keycloak realm and gives testkit an operator client certificate:

```shell
nix run .#openshell-test-runner -- run \
--environment ubuntu-docker-rootful-oidc --installer deb \
--family features --package oidc-auth
```

A suite that needs setup beyond a container engine runs in a feature
environment, named `<base>-<feature>`, such as `ubuntu-docker-rootful-keycloak`
or `ubuntu-docker-rootful-cdi`. `features` in `tests/tmachine/config.nix` lists
Expand Down Expand Up @@ -631,11 +642,27 @@ nix run .#openshell-test-runner -- run \

#### Gateway overrides in an environment

An environment overrides gateway settings by writing the tables to change to
`/etc/openshell-test/gateway-overrides.toml` in a playbook that runs before the
installer. The file holds whole tables, such as `[openshell.gateway]` or
`[openshell.drivers.podman]`, and is not a complete configuration. Each
installer applies it to the gateway's own file:
An environment overrides gateway settings with the `gateway_overrides` role,
which takes a name and the tables to change, such as `[openshell.gateway]` or
`[openshell.drivers.podman]`:

```yaml
- name: Push configuration to supervisors
ansible.builtin.include_role:
name: gateway_overrides
vars:
gateway_overrides_name: config-push
gateway_overrides_content: |
[openshell.gateway]
config_delivery_mode = "push"
```

The role writes each name as its own marked block in
`/etc/openshell-test/gateway-overrides.toml`, so writers keep each other's
settings, and `gateway_overrides_state: absent` removes a name's block.
Two blocks must not declare the same table. Don't write the file
directly. The file is not a complete configuration. Each installer applies it
to the gateway's own file:

- `binaries` applies it to the `/etc/openshell/gateway.toml` it generates.
- `deb` and `rpm` apply it to the service user's
Expand Down Expand Up @@ -690,12 +717,29 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on
the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to
the node's address.

The `routable-host-ip` feature suite (`tests/suites/features/routable-host-ip`)
checks that a policy endpoint whose host is an IP literal reaches that address,
with or without `allowed_ips`. The test reaches a host server by
`host.openshell.internal`, then uses the address that request arrived at, or
the host's outbound address when it arrived on loopback from a host-network
supervisor. The address must be one the supervisor lets an IP literal reach.
Rootless Podman's is link-local and always blocked, so CI runs the suite on
Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` installers.

The `connect-ssh-supervision` feature suite
(`tests/suites/features/connect-ssh-supervision`) checks how `sandbox connect`
supervises its `ssh` process: it reconnects after losing the SSH transport and
forwards a SIGTERM to `ssh`. The tests find those processes in `/proc`, so they
need a Linux test host. CI runs them on Docker rootful.

The `service-exposure` feature suite (`tests/suites/features/service-exposure`)
exposes a service in a sandbox and checks how the gateway forwards the client's
`Authorization` header. The test sends requests to the gateway's endpoint with
the service URL's host as the Host header, so the test machine doesn't resolve
service hosts. CI runs it on Docker rootful and Podman rootful and rootless. The
Helm chart turns gateway service routing off, so the K3s installers have no
service URLs.

Run the VM-backed Rust CLI e2e suite:

```shell
Expand Down
2 changes: 0 additions & 2 deletions e2e/rust/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

37 changes: 0 additions & 37 deletions e2e/rust/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,6 @@ name = "policy_activation"
path = "tests/policy_activation.rs"
required-features = ["e2e-docker"]

[[test]]
name = "provider_files"
path = "tests/provider_files.rs"
required-features = ["e2e-docker"]

[[test]]
name = "oidc_pkce"
path = "tests/oidc_pkce.rs"
Expand All @@ -78,11 +73,6 @@ name = "vm_overlay"
path = "tests/vm_overlay.rs"
required-features = ["e2e-vm"]

[[test]]
name = "service_bearer_passthrough"
path = "tests/service_bearer_passthrough.rs"
required-features = ["e2e-docker"]

[[test]]
name = "rootfs_tar"
path = "tests/rootfs_tar.rs"
Expand Down Expand Up @@ -198,11 +188,6 @@ name = "credential_drivers"
path = "tests/credential_drivers.rs"
required-features = ["e2e-kubernetes-credential-drivers"]

[[test]]
name = "websocket_conformance"
path = "tests/websocket_conformance.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "user_namespaces"
path = "tests/user_namespaces.rs"
Expand All @@ -213,31 +198,11 @@ name = "host_gateway_alias"
path = "tests/host_gateway_alias.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "forward_proxy_l7_bypass"
path = "tests/forward_proxy_l7_bypass.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "forward_proxy_graphql_l7"
path = "tests/forward_proxy_graphql_l7.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "forward_proxy_jsonrpc_l7"
path = "tests/forward_proxy_jsonrpc_l7.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "supervisor_middleware_http_hooks"
path = "tests/supervisor_middleware_http_hooks.rs"
required-features = ["e2e-docker"]

[[test]]
name = "proxy_egress_pipeline"
path = "tests/proxy_egress_pipeline.rs"
required-features = ["e2e-host-gateway"]

[[test]]
name = "workspace_namespace_managed"
path = "tests/workspace_namespace_managed.rs"
Expand Down Expand Up @@ -279,13 +244,11 @@ sha1 = "0.10"
sha2 = "0.10"
hex = "0.4"
rand = "0.9"
rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] }
tonic = { version = "0.14", features = ["transport"] }
tonic-prost = "0.14"
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] }
tower = "0.5"
url = "2"

Expand Down
6 changes: 0 additions & 6 deletions e2e/rust/e2e-podman.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,11 @@ source "${ROOT}/e2e/support/conformance.sh"
# unfiltered e2e:podman task remains available while the omitted targets are
# stabilized and can be added here.
PODMAN_CI_TESTS=(
bypass_detection
ephemeral_cleanup
driver_config_volume
edge_tunnel_e2e
forward_proxy_graphql_l7
forward_proxy_jsonrpc_l7
forward_proxy_l7_bypass
host_gateway_alias
live_policy_update
local_driver_token_restart
no_proxy
ocsf_jsonl_conformance
otlp_relay
podman_corporate_proxy
Expand Down
20 changes: 0 additions & 20 deletions e2e/rust/src/harness/sandbox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -566,26 +566,6 @@ impl SandboxGuard {
Ok(combined)
}

/// Spawn `openshell forward start` as a background process.
///
/// Returns the child process handle. The caller is responsible for killing
/// it (or it will be killed on drop since `kill_on_drop(true)` is set).
///
/// # Errors
///
/// Returns an error if the process cannot be spawned.
pub fn spawn_forward(&self, port: u16) -> Result<tokio::process::Child, String> {
let mut cmd = openshell_cmd();
cmd.arg("forward")
.arg("start")
.arg(port.to_string())
.arg(&self.name);
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());

cmd.spawn()
.map_err(|e| format!("failed to spawn port forward: {e}"))
}

/// Delete the sandbox explicitly.
///
/// Also kills the background child process if one exists. This is called
Expand Down
79 changes: 0 additions & 79 deletions e2e/rust/tests/bypass_detection.rs

This file was deleted.

Loading
Loading