From 137461798d5eab85119e4dcd72aade98c678ff61 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 6 Nov 2026 21:59:28 +1100 Subject: [PATCH 1/2] test(stack-kms-vendor): share the vendor fixture harness The fixture types, the fixture loader, body_json and the Replay transport were in each of the four vendor crates' tests/common/mod.rs. The copies were almost the same, but they had started to differ: only the Azure Replay kept its waits, and only the AWS copy sent a null body as "null". Move the harness to stack_kms_vendor::replay, behind the test-support feature. That feature already holds the contract suite and FakeVendor, which every vendor crate uses as a dev-dependency, so this needs no new crate and no new dependency edge. The module also holds the Recording transport that the Vault crate uses to write fixtures. Each tests/common/mod.rs now keeps only its vendor parts: the emulator, the test keys and the constants. The fixture files do not change. The feature now pulls in serde for the fixture types. The @cipherstash/auth binding does not turn on test-support, so its binary does not change. The changeset records the src/ change, as the repo rule requires. --- .changeset/kms-vendor-shared-test-harness.md | 5 + AGENTS.md | 2 +- Cargo.lock | 1 + packages/stack-kms-aws/tests/common/mod.rs | 124 +-------- packages/stack-kms-azure/tests/common/mod.rs | 147 ++-------- packages/stack-kms-gcp/tests/common/mod.rs | 140 ++-------- packages/stack-kms-vault/tests/common/mod.rs | 198 ++------------ packages/stack-kms-vendor/Cargo.toml | 10 +- packages/stack-kms-vendor/src/lib.rs | 2 + packages/stack-kms-vendor/src/replay.rs | 272 +++++++++++++++++++ 10 files changed, 351 insertions(+), 550 deletions(-) create mode 100644 .changeset/kms-vendor-shared-test-harness.md create mode 100644 packages/stack-kms-vendor/src/replay.rs diff --git a/.changeset/kms-vendor-shared-test-harness.md b/.changeset/kms-vendor-shared-test-harness.md new file mode 100644 index 000000000..ed3f4fdad --- /dev/null +++ b/.changeset/kms-vendor-shared-test-harness.md @@ -0,0 +1,5 @@ +--- +'@cipherstash/auth': patch +--- + +No change to the binding. The `stack-kms-vendor` crate that it compiles in has a new test-only module, `replay`, behind its `test-support` feature. The binding does not turn on that feature, so the module is not in the binary. diff --git a/AGENTS.md b/AGENTS.md index 8e493b6a6..9a78a3c4b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,7 +91,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l `cipherstash/encrypt-query-language` repository. Old upstream issue and PR links are provenance only. - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". -- `packages/stack-kms-vendor`, `packages/stack-kms-aws`, `packages/stack-kms-vault`, `packages/stack-kms-gcp`, `packages/stack-kms-azure`: The key vendor backends (ADR-0009 to ADR-0011 in `packages/stack-encrypt/docs/adr/`). `stack-kms-vendor` is the vendor seam (`KeyVendorBackend`), keyset records, both key modes, the static key backend and the `VendorTransport` HTTP seam (feature `http`: the native reqwest transport; the WASI guest's transport is `stack-guest-abi`'s `vendor` feature). `stack-kms-aws` is the AWS KMS backend: a hand-written JSON client with SigV4 signing in-crate and the AWS credential providers; its LocalStack test runs in `tests-crates.yml`'s `localstack` job. `stack-kms-vault` is the HashiCorp Vault Transit backend: a hand-written Transit client with `batch_input` calls and token, AppRole and Kubernetes auth; its `vault server -dev` test runs in `tests-crates.yml`'s `vault` job. `stack-kms-gcp` is the Google Cloud KMS backend: a hand-written Cloud KMS REST client with CRC32C checks, service account (RS256 JWT, the pure-Rust `rsa` crate, which has the open advisory RUSTSEC-2023-0071) and metadata server credentials; the service account login is the default feature `service-account`, and the Go guest and `@cipherstash/auth` build the crate without it, so they hold no RSA code; Google has no Cloud KMS emulator, so its live test runs only in the opt-in, credentialed `integration-gcp-kms.yml`, which is inert until the `STACK_KMS_GCP_LIVE_KEY` repository variable is set. `stack-kms-azure` is the Azure Key Vault and Managed HSM backend: a hand-written Key Vault REST client (`wrapkey`/`unwrapkey`, RSA-OAEP-256 or A256KW, the associated data bound by `stack_kms_vendor::bound` because Key Vault has none) with Entra ID client secret, workload identity and managed identity credentials, and no RSA code of its own; Microsoft has no Key Vault emulator, so its live test runs only in the opt-in, credentialed `integration-azure-key-vault.yml`, which is inert until the `STACK_KMS_AZURE_LIVE_KEY` repository variable is set. All five are root workspace members, publishable in the `stack-kms-vendor` version group, and staged in `release-plz.toml` (`release = false`) until their first version is published by hand. The `stack-kms-vendor` crate docs hold the capability matrix of every key backend (key lifetime, vendor calls, what the vendor sees and can refuse, credentials, rotation, known limits); change it with the code it describes, and keep the short matrix in the Go `encrypt` package docs in step. The `@cipherstash/auth` binding compiles all five in for `stash keyset`, so a change under their `src/` needs an `@cipherstash/auth` changeset too. TypeScript (`@cipherstash/stack`) cannot use a key vendor backend yet. +- `packages/stack-kms-vendor`, `packages/stack-kms-aws`, `packages/stack-kms-vault`, `packages/stack-kms-gcp`, `packages/stack-kms-azure`: The key vendor backends (ADR-0009 to ADR-0011 in `packages/stack-encrypt/docs/adr/`). `stack-kms-vendor` is the vendor seam (`KeyVendorBackend`), keyset records, both key modes, the static key backend and the `VendorTransport` HTTP seam (feature `http`: the native reqwest transport; the WASI guest's transport is `stack-guest-abi`'s `vendor` feature). Its `test-support` feature holds the test code every vendor crate shares: the contract suite (`contract`), `FakeVendor`, and the recorded-fixture harness (`replay`: fixture types, `Replay`, `Recording`); a vendor crate's `tests/common/mod.rs` holds only its own emulator, keys and constants. `stack-kms-aws` is the AWS KMS backend: a hand-written JSON client with SigV4 signing in-crate and the AWS credential providers; its LocalStack test runs in `tests-crates.yml`'s `localstack` job. `stack-kms-vault` is the HashiCorp Vault Transit backend: a hand-written Transit client with `batch_input` calls and token, AppRole and Kubernetes auth; its `vault server -dev` test runs in `tests-crates.yml`'s `vault` job. `stack-kms-gcp` is the Google Cloud KMS backend: a hand-written Cloud KMS REST client with CRC32C checks, service account (RS256 JWT, the pure-Rust `rsa` crate, which has the open advisory RUSTSEC-2023-0071) and metadata server credentials; the service account login is the default feature `service-account`, and the Go guest and `@cipherstash/auth` build the crate without it, so they hold no RSA code; Google has no Cloud KMS emulator, so its live test runs only in the opt-in, credentialed `integration-gcp-kms.yml`, which is inert until the `STACK_KMS_GCP_LIVE_KEY` repository variable is set. `stack-kms-azure` is the Azure Key Vault and Managed HSM backend: a hand-written Key Vault REST client (`wrapkey`/`unwrapkey`, RSA-OAEP-256 or A256KW, the associated data bound by `stack_kms_vendor::bound` because Key Vault has none) with Entra ID client secret, workload identity and managed identity credentials, and no RSA code of its own; Microsoft has no Key Vault emulator, so its live test runs only in the opt-in, credentialed `integration-azure-key-vault.yml`, which is inert until the `STACK_KMS_AZURE_LIVE_KEY` repository variable is set. All five are root workspace members, publishable in the `stack-kms-vendor` version group, and staged in `release-plz.toml` (`release = false`) until their first version is published by hand. The `stack-kms-vendor` crate docs hold the capability matrix of every key backend (key lifetime, vendor calls, what the vendor sees and can refuse, credentials, rotation, known limits); change it with the code it describes, and keep the short matrix in the Go `encrypt` package docs in step. The `@cipherstash/auth` binding compiles all five in for `stash keyset`, so a change under their `src/` needs an `@cipherstash/auth` changeset too. TypeScript (`@cipherstash/stack`) cannot use a key vendor backend yet. - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate and the five key vendor crates included (the binding compiles them in for `keysetCreate` and `keysetRewrap`, which `stash keyset` calls), needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) diff --git a/Cargo.lock b/Cargo.lock index d3b99f6df..364fd8de6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3735,6 +3735,7 @@ dependencies = [ "hkdf", "miette", "reqwest", + "serde", "serde_json", "sha2 0.11.0", "stack-auth", diff --git a/packages/stack-kms-aws/tests/common/mod.rs b/packages/stack-kms-aws/tests/common/mod.rs index b9b2ab672..4560273b7 100644 --- a/packages/stack-kms-aws/tests/common/mod.rs +++ b/packages/stack-kms-aws/tests/common/mod.rs @@ -1,18 +1,22 @@ -//! Shared test support: recorded fixtures, a replaying transport, and an -//! in-process emulation of the AWS KMS JSON API. +//! Shared test support: the fixture time and credentials, and an in-process +//! emulation of the AWS KMS JSON API. The fixture and replay harness is +//! `stack_kms_vendor::replay`, shared by every vendor crate. #![allow(dead_code, clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, Mutex}; use std::time::Duration; use aes_gcm::aead::{Aead, KeyInit, Payload}; use aes_gcm::{Aes256Gcm, Nonce}; use base64ct::{Base64, Encoding}; -use serde::Deserialize; +use serde::de::DeserializeOwned; use serde_json::{json, Value}; use stack_kms_aws::{AwsKms, AwsKmsConfig, Clock, Credentials}; +// Each test binary uses a different part of the harness. +#[allow(unused_imports)] +pub use stack_kms_vendor::replay::{Fixture, FixtureRequest, FixtureResponse, Fixtures, Replay}; use stack_kms_vendor::{TransportError, VendorRequest, VendorResponse, VendorTransport}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -35,114 +39,12 @@ pub fn fixture_credentials() -> Credentials { ) } -#[derive(Deserialize)] -pub struct FixtureRequest { - pub method: String, - pub url: String, - pub headers: Vec<(String, String)>, - pub body: Value, -} - -#[derive(Clone, Deserialize)] -pub struct FixtureResponse { - pub status: u16, - #[serde(default)] - pub headers: Vec<(String, String)>, - pub body: Value, -} - -impl FixtureResponse { - pub fn to_response(&self) -> VendorResponse { - let body = match &self.body { - Value::String(text) => text.clone().into_bytes(), - other => serde_json::to_vec(other).unwrap(), - }; - let mut response = VendorResponse::new(self.status, body); - for (name, value) in &self.headers { - response = response.header(name, value); - } - response - } -} - -#[derive(Deserialize)] -pub struct Fixture { - pub request: FixtureRequest, - pub response: FixtureResponse, -} - -pub fn load Deserialize<'de>>(name: &str) -> T { - let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")); - let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); - serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}")) -} - -/// A transport that checks each request against a recorded one and -/// answers with the recorded response. -#[derive(Default)] -pub struct Replay { - exchanges: Mutex, VendorResponse)>>, -} - -impl Replay { - pub fn new() -> Self { - Self::default() - } - - /// Expect `fixture`'s request and answer with its response. - pub fn expect(self, fixture: Fixture) -> Self { - let response = fixture.response.to_response(); - self.exchanges - .lock() - .unwrap() - .push_back((Some(fixture.request), response)); - self - } - - /// Answer the next request, whatever it is, with `response`. - pub fn answer(self, response: VendorResponse) -> Self { - self.exchanges.lock().unwrap().push_back((None, response)); - self - } - - pub fn is_done(&self) -> bool { - self.exchanges.lock().unwrap().is_empty() - } -} - -impl VendorTransport for Replay { - async fn send(&self, request: VendorRequest) -> Result { - let (expected, response) = self - .exchanges - .lock() - .unwrap() - .pop_front() - .expect("a request the fixtures do not expect"); - if let Some(expected) = expected { - let body: Value = serde_json::from_slice(&request.body).unwrap(); - let actual = json!({ - "method": request.method, - "url": request.url, - "headers": request.headers, - "body": body, - }); - let wanted = json!({ - "method": expected.method, - "url": expected.url, - "headers": expected.headers, - "body": expected.body, - }); - assert_eq!( - actual, - wanted, - "the request differs from the fixture; actual:\n{}", - serde_json::to_string_pretty(&actual).unwrap() - ); - } - Ok(response) - } +/// The fixtures of this crate. +pub const FIXTURES: Fixtures = + Fixtures::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")); - async fn sleep(&self, _duration: Duration) {} +pub fn load(name: &str) -> T { + FIXTURES.load(name) } /// A client over `transport` with the fixture time and credentials. diff --git a/packages/stack-kms-azure/tests/common/mod.rs b/packages/stack-kms-azure/tests/common/mod.rs index 3a53a9f84..3dbd1a1d2 100644 --- a/packages/stack-kms-azure/tests/common/mod.rs +++ b/packages/stack-kms-azure/tests/common/mod.rs @@ -1,10 +1,11 @@ -//! Shared test support: fixtures, a replaying transport, the test-only RSA -//! key, an RFC 3394 key wrap, and an in-process emulation of the Key Vault -//! and Managed HSM key API and the Entra ID token endpoint. +//! Shared test support: the test-only RSA key, an RFC 3394 key wrap, and an +//! in-process emulation of the Key Vault and Managed HSM key API and the +//! Entra ID token endpoint. The fixture and replay harness is +//! `stack_kms_vendor::replay`, shared by every vendor crate. #![allow(dead_code, clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -15,9 +16,13 @@ use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use rsa::pkcs8::DecodePrivateKey; use rsa::sha2::Sha256; use rsa::{Oaep, RsaPrivateKey}; -use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; use serde_json::{json, Value}; use stack_kms_azure::{AccessToken, AzureKeyVault, AzureKeyVaultConfig, KeyId}; +use stack_kms_vendor::replay::body_json; +// Each test binary uses a different part of the harness. +#[allow(unused_imports)] +pub use stack_kms_vendor::replay::{Fixture, FixtureRequest, FixtureResponse, Fixtures, Replay}; use stack_kms_vendor::{TransportError, VendorRequest, VendorResponse, VendorTransport}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -123,136 +128,16 @@ pub fn kw_unwrap(vendor_key: &[u8; 32], ciphertext: &[u8]) -> Option> { (a == KW_IV).then(|| r.concat()) } -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureRequest { - pub method: String, - pub url: String, - pub headers: Vec<(String, String)>, - pub body: Value, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureResponse { - pub status: u16, - #[serde(default)] - pub headers: Vec<(String, String)>, - pub body: Value, -} - -impl FixtureResponse { - pub fn to_response(&self) -> VendorResponse { - let body = match &self.body { - Value::String(text) => text.clone().into_bytes(), - Value::Null => Vec::new(), - other => serde_json::to_vec(other).unwrap(), - }; - let mut response = VendorResponse::new(self.status, body); - for (name, value) in &self.headers { - response = response.header(name, value); - } - response - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Fixture { - pub request: FixtureRequest, - pub response: FixtureResponse, -} - -pub fn load Deserialize<'de>>(name: &str) -> T { - let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")); - let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); - serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}")) -} - -fn body_json(body: &[u8]) -> Value { - if body.is_empty() { - Value::Null - } else { - serde_json::from_slice(body) - .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(body).into())) - } -} - -/// A transport that checks each request against a fixed one and answers -/// with the fixed response. -#[derive(Default)] -pub struct Replay { - exchanges: Mutex, VendorResponse)>>, - sleeps: Mutex>, -} - -impl Replay { - pub fn new() -> Self { - Self::default() - } +/// The fixtures of this crate. +pub const FIXTURES: Fixtures = + Fixtures::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")); - /// Expect each exchange of a fixture file, in order. - pub fn expect_all(self, fixtures: Vec) -> Self { - for fixture in fixtures { - let response = fixture.response.to_response(); - self.exchanges - .lock() - .unwrap() - .push_back((Some(fixture.request), response)); - } - self - } - - /// Answer the next request, whatever it is, with `response`. - pub fn answer(self, response: VendorResponse) -> Self { - self.exchanges.lock().unwrap().push_back((None, response)); - self - } - - pub fn is_done(&self) -> bool { - self.exchanges.lock().unwrap().is_empty() - } - - pub fn sleeps(&self) -> Vec { - self.sleeps.lock().unwrap().clone() - } -} - -impl VendorTransport for Replay { - async fn send(&self, request: VendorRequest) -> Result { - let (expected, response) = self - .exchanges - .lock() - .unwrap() - .pop_front() - .expect("a request the fixtures do not expect"); - if let Some(expected) = expected { - let actual = json!({ - "method": request.method, - "url": request.url, - "headers": request.headers, - "body": body_json(&request.body), - }); - let wanted = json!({ - "method": expected.method, - "url": expected.url, - "headers": expected.headers, - "body": expected.body, - }); - assert_eq!( - actual, - wanted, - "the request differs from the fixture; actual:\n{}", - serde_json::to_string_pretty(&actual).unwrap() - ); - } - Ok(response) - } - - async fn sleep(&self, duration: Duration) { - self.sleeps.lock().unwrap().push(duration); - } +pub fn load(name: &str) -> T { + FIXTURES.load(name) } pub fn replay(name: &str) -> Replay { - Replay::new().expect_all(load(name)) + FIXTURES.replay(name) } /// A client with the default configuration over `transport`, with the diff --git a/packages/stack-kms-gcp/tests/common/mod.rs b/packages/stack-kms-gcp/tests/common/mod.rs index e4c507c7a..45afc57f0 100644 --- a/packages/stack-kms-gcp/tests/common/mod.rs +++ b/packages/stack-kms-gcp/tests/common/mod.rs @@ -1,10 +1,11 @@ -//! Shared test support: fixtures, a replaying transport, the test-only -//! service account, and an in-process emulation of the Cloud KMS API, the -//! token endpoint and the metadata server. +//! Shared test support: the test-only service account, and an in-process +//! emulation of the Cloud KMS API, the token endpoint and the metadata +//! server. The fixture and replay harness is `stack_kms_vendor::replay`, +//! shared by every vendor crate. #![allow(dead_code, clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -15,10 +16,14 @@ use rsa::pkcs1v15::{Signature, VerifyingKey}; use rsa::pkcs8::DecodePrivateKey; use rsa::sha2::Sha256; use rsa::signature::Verifier; -use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; use serde_json::{json, Value}; use stack_kms_gcp::crc32c::crc32c; use stack_kms_gcp::{parse_key_name, AccessToken, GcpKms, GcpKmsConfig, ServiceAccountKey}; +use stack_kms_vendor::replay::body_json; +// Each test binary uses a different part of the harness. +#[allow(unused_imports)] +pub use stack_kms_vendor::replay::{Fixture, FixtureRequest, FixtureResponse, Fixtures, Replay}; use stack_kms_vendor::{TransportError, VendorRequest, VendorResponse, VendorTransport}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -107,129 +112,16 @@ pub fn verify_jwt(jwt: &str) -> Option<(Value, Value)> { Some((header, claims)) } -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureRequest { - pub method: String, - pub url: String, - pub headers: Vec<(String, String)>, - pub body: Value, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureResponse { - pub status: u16, - #[serde(default)] - pub headers: Vec<(String, String)>, - pub body: Value, -} - -impl FixtureResponse { - pub fn to_response(&self) -> VendorResponse { - let body = match &self.body { - Value::String(text) => text.clone().into_bytes(), - Value::Null => Vec::new(), - other => serde_json::to_vec(other).unwrap(), - }; - let mut response = VendorResponse::new(self.status, body); - for (name, value) in &self.headers { - response = response.header(name, value); - } - response - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Fixture { - pub request: FixtureRequest, - pub response: FixtureResponse, -} - -pub fn load Deserialize<'de>>(name: &str) -> T { - let path = format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")); - let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); - serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}")) -} - -fn body_json(body: &[u8]) -> Value { - if body.is_empty() { - Value::Null - } else { - serde_json::from_slice(body) - .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(body).into())) - } -} - -/// A transport that checks each request against a recorded one and -/// answers with the recorded response. -#[derive(Default)] -pub struct Replay { - exchanges: Mutex, VendorResponse)>>, -} - -impl Replay { - pub fn new() -> Self { - Self::default() - } - - /// Expect each exchange of a fixture file, in order. - pub fn expect_all(self, fixtures: Vec) -> Self { - for fixture in fixtures { - let response = fixture.response.to_response(); - self.exchanges - .lock() - .unwrap() - .push_back((Some(fixture.request), response)); - } - self - } - - /// Answer the next request, whatever it is, with `response`. - pub fn answer(self, response: VendorResponse) -> Self { - self.exchanges.lock().unwrap().push_back((None, response)); - self - } - - pub fn is_done(&self) -> bool { - self.exchanges.lock().unwrap().is_empty() - } -} - -impl VendorTransport for Replay { - async fn send(&self, request: VendorRequest) -> Result { - let (expected, response) = self - .exchanges - .lock() - .unwrap() - .pop_front() - .expect("a request the fixtures do not expect"); - if let Some(expected) = expected { - let actual = json!({ - "method": request.method, - "url": request.url, - "headers": request.headers, - "body": body_json(&request.body), - }); - let wanted = json!({ - "method": expected.method, - "url": expected.url, - "headers": expected.headers, - "body": expected.body, - }); - assert_eq!( - actual, - wanted, - "the request differs from the fixture; actual:\n{}", - serde_json::to_string_pretty(&actual).unwrap() - ); - } - Ok(response) - } +/// The fixtures of this crate. +pub const FIXTURES: Fixtures = + Fixtures::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")); - async fn sleep(&self, _duration: Duration) {} +pub fn load(name: &str) -> T { + FIXTURES.load(name) } pub fn replay(name: &str) -> Replay { - Replay::new().expect_all(load(name)) + FIXTURES.replay(name) } /// A client of the public endpoint over `transport`, with the fixture diff --git a/packages/stack-kms-vault/tests/common/mod.rs b/packages/stack-kms-vault/tests/common/mod.rs index 63b5852ad..8358f91a6 100644 --- a/packages/stack-kms-vault/tests/common/mod.rs +++ b/packages/stack-kms-vault/tests/common/mod.rs @@ -1,20 +1,27 @@ -//! Shared test support: recorded fixtures, a replaying transport, a -//! recording transport, and an in-process emulation of the Vault Transit -//! API. +//! Shared test support: the fixture address, token and keys, and an +//! in-process emulation of the Vault Transit API. The fixture, replay and +//! recording harness is `stack_kms_vendor::replay`, shared by every vendor +//! crate. #![allow(dead_code, clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::sync::{Arc, Mutex}; use std::time::Duration; use aes_gcm::aead::{Aead, KeyInit}; use aes_gcm::{Aes256Gcm, Nonce}; use base64ct::{Base64, Encoding}; -use serde::{Deserialize, Serialize}; +use serde::de::DeserializeOwned; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; use stack_kms_vault::{VaultConfig, VaultToken, VaultTransit}; +use stack_kms_vendor::replay::body_json; +// Each test binary uses a different part of the harness. +#[allow(unused_imports)] +pub use stack_kms_vendor::replay::{ + Fixture, FixtureRequest, FixtureResponse, Fixtures, Recording, Replay, +}; use stack_kms_vendor::{TransportError, VendorRequest, VendorResponse, VendorTransport}; /// The address of every fixture: a local `vault server -dev`. @@ -26,185 +33,16 @@ pub const KEY: &str = "cipherstash-fixture"; /// The derived Transit key of the fixtures. pub const DERIVED_KEY: &str = "cipherstash-fixture-derived"; -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureRequest { - pub method: String, - pub url: String, - pub headers: Vec<(String, String)>, - pub body: Value, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct FixtureResponse { - pub status: u16, - #[serde(default)] - pub headers: Vec<(String, String)>, - pub body: Value, -} - -impl FixtureResponse { - pub fn to_response(&self) -> VendorResponse { - let body = match &self.body { - Value::String(text) => text.clone().into_bytes(), - Value::Null => Vec::new(), - other => serde_json::to_vec(other).unwrap(), - }; - let mut response = VendorResponse::new(self.status, body); - for (name, value) in &self.headers { - response = response.header(name, value); - } - response - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Fixture { - pub request: FixtureRequest, - pub response: FixtureResponse, -} +/// The fixtures of this crate. +pub const FIXTURES: Fixtures = + Fixtures::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")); pub fn fixture_path(name: &str) -> String { - format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) -} - -pub fn load Deserialize<'de>>(name: &str) -> T { - let path = fixture_path(name); - let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); - serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}")) + FIXTURES.path(name) } -fn body_json(body: &[u8]) -> Value { - if body.is_empty() { - Value::Null - } else { - serde_json::from_slice(body) - .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(body).into())) - } -} - -/// A transport that checks each request against a recorded one and -/// answers with the recorded response. -#[derive(Default)] -pub struct Replay { - exchanges: Mutex, VendorResponse)>>, -} - -impl Replay { - pub fn new() -> Self { - Self::default() - } - - /// Expect `fixture`'s request and answer with its response. - pub fn expect(self, fixture: Fixture) -> Self { - let response = fixture.response.to_response(); - self.exchanges - .lock() - .unwrap() - .push_back((Some(fixture.request), response)); - self - } - - /// Expect each exchange of a recorded sequence, in order. - pub fn expect_all(mut self, fixtures: Vec) -> Self { - for fixture in fixtures { - self = self.expect(fixture); - } - self - } - - /// Answer the next request, whatever it is, with `response`. - pub fn answer(self, response: VendorResponse) -> Self { - self.exchanges.lock().unwrap().push_back((None, response)); - self - } - - pub fn is_done(&self) -> bool { - self.exchanges.lock().unwrap().is_empty() - } -} - -impl VendorTransport for Replay { - async fn send(&self, request: VendorRequest) -> Result { - let (expected, response) = self - .exchanges - .lock() - .unwrap() - .pop_front() - .expect("a request the fixtures do not expect"); - if let Some(expected) = expected { - let actual = json!({ - "method": request.method, - "url": request.url, - "headers": request.headers, - "body": body_json(&request.body), - }); - let wanted = json!({ - "method": expected.method, - "url": expected.url, - "headers": expected.headers, - "body": expected.body, - }); - assert_eq!( - actual, - wanted, - "the request differs from the fixture; actual:\n{}", - serde_json::to_string_pretty(&actual).unwrap() - ); - } - Ok(response) - } - - async fn sleep(&self, _duration: Duration) {} -} - -/// A transport that sends through `inner` and keeps every exchange, to -/// write fixtures. -pub struct Recording { - pub inner: T, - pub exchanges: Mutex>, -} - -impl Recording { - pub fn new(inner: T) -> Self { - Self { - inner, - exchanges: Mutex::new(Vec::new()), - } - } - - /// The exchanges so far, and forget them. - pub fn take(&self) -> Vec { - std::mem::take(&mut *self.exchanges.lock().unwrap()) - } -} - -impl VendorTransport for Recording { - async fn send(&self, request: VendorRequest) -> Result { - let recorded = FixtureRequest { - method: request.method.clone(), - url: request.url.clone(), - headers: request.headers.clone(), - body: body_json(&request.body), - }; - let response = self.inner.send(request).await?; - let headers = response - .header_value("content-type") - .map(|value| vec![("content-type".to_owned(), value.to_owned())]) - .unwrap_or_default(); - self.exchanges.lock().unwrap().push(Fixture { - request: recorded, - response: FixtureResponse { - status: response.status, - headers, - body: body_json(&response.body), - }, - }); - Ok(response) - } - - async fn sleep(&self, duration: Duration) { - self.inner.sleep(duration).await; - } +pub fn load(name: &str) -> T { + FIXTURES.load(name) } /// A client over `transport` with the fixture address and token. diff --git a/packages/stack-kms-vendor/Cargo.toml b/packages/stack-kms-vendor/Cargo.toml index 8e0d77a9a..f133b4199 100644 --- a/packages/stack-kms-vendor/Cargo.toml +++ b/packages/stack-kms-vendor/Cargo.toml @@ -17,9 +17,11 @@ license-file = "LICENSE" [features] # `stack_kms_vendor::contract` (the shared contract suite every key vendor -# backend runs) and `FakeVendor` (an in-memory key vendor backend). The -# contract suite drives a real `StackCipher`, so this pulls stack-encrypt in. -test-support = ["dep:stack-encrypt", "stack-kms/test-support"] +# backend runs), `FakeVendor` (an in-memory key vendor backend) and +# `stack_kms_vendor::replay` (the recorded-fixture harness every vendor +# crate's tests use). The contract suite drives a real `StackCipher`, so this +# pulls stack-encrypt in; the fixtures are serde types, so it pulls serde in. +test-support = ["dep:stack-encrypt", "dep:serde", "stack-kms/test-support"] # `HttpVendorTransport`: the native `VendorTransport`, over the reqwest client # and the TLS configuration stack-kms uses for ZeroKMS. Native only: on wasm32 # the dependencies are not in the graph and the type does not exist, because @@ -39,6 +41,8 @@ hkdf = "0.13" sha2 = "0.11" subtle = "2.6" miette = { workspace = true } +# The fixture types of `replay` only. +serde = { workspace = true, optional = true } serde_json = { workspace = true } thiserror = { workspace = true } uuid = { workspace = true } diff --git a/packages/stack-kms-vendor/src/lib.rs b/packages/stack-kms-vendor/src/lib.rs index 78fdf1d5e..90e776208 100644 --- a/packages/stack-kms-vendor/src/lib.rs +++ b/packages/stack-kms-vendor/src/lib.rs @@ -251,6 +251,8 @@ mod vendor; pub mod contract; #[cfg(feature = "test-support")] mod fake; +#[cfg(feature = "test-support")] +pub mod replay; pub use batch::{run_batch, BatchOptions}; pub use error::RecordError; diff --git a/packages/stack-kms-vendor/src/replay.rs b/packages/stack-kms-vendor/src/replay.rs new file mode 100644 index 000000000..31efdcfdc --- /dev/null +++ b/packages/stack-kms-vendor/src/replay.rs @@ -0,0 +1,272 @@ +//! Recorded HTTP exchanges for the tests of a key vendor backend. +//! +//! Every vendor crate tests its client against fixture files: JSON that +//! holds one exchange ([`Fixture`]) or a sequence of them. This module is +//! the one copy of the harness that reads them: +//! +//! * [`Fixtures`] finds and parses the files of one crate; +//! * [`Replay`] is a [`VendorTransport`] that checks each request against a +//! fixture and answers with the fixture's response; +//! * [`Recording`] is a [`VendorTransport`] that sends through another one +//! and keeps every exchange, to write fixtures from a live service. +//! +//! The vendor-specific parts (emulators, keys, constants) stay in each +//! crate's `tests/common/mod.rs`. +//! +//! ```ignore +//! use stack_kms_vendor::replay::{Fixtures, Replay}; +//! +//! const FIXTURES: Fixtures = +//! Fixtures::new(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")); +//! +//! let transport = Replay::new().expect(FIXTURES.load("encrypt.json")); +//! ``` + +// The harness is test code: it panics when a request differs from its +// fixture or a fixture file does not parse. +#![allow(clippy::panic, clippy::unwrap_used, clippy::expect_used)] + +use std::collections::VecDeque; +use std::sync::Mutex; +use std::time::Duration; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +use crate::{TransportError, VendorRequest, VendorResponse, VendorTransport}; + +/// The request half of a recorded exchange. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct FixtureRequest { + /// The HTTP method. + pub method: String, + /// The full URL. + pub url: String, + /// Every header, in the order it is sent. + pub headers: Vec<(String, String)>, + /// The body as JSON; see [`body_json`]. + pub body: Value, +} + +/// The response half of a recorded exchange. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct FixtureResponse { + /// The HTTP status. + pub status: u16, + /// The headers. Missing means none. + #[serde(default)] + pub headers: Vec<(String, String)>, + /// The body: a JSON string is sent as its text, `null` as no bytes, + /// and any other value as its JSON form. + pub body: Value, +} + +impl FixtureResponse { + /// The response this fixture describes. + pub fn to_response(&self) -> VendorResponse { + let body = match &self.body { + Value::String(text) => text.clone().into_bytes(), + Value::Null => Vec::new(), + other => serde_json::to_vec(other).unwrap(), + }; + let mut response = VendorResponse::new(self.status, body); + for (name, value) in &self.headers { + response = response.header(name, value); + } + response + } +} + +/// One recorded exchange: a request and the response to it. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Fixture { + /// The request the client must send. + pub request: FixtureRequest, + /// The response the vendor gave. + pub response: FixtureResponse, +} + +/// The fixture directory of one crate. +#[derive(Clone, Copy, Debug)] +pub struct Fixtures { + dir: &'static str, +} + +impl Fixtures { + /// The fixtures in `dir`. Give an absolute path, such as + /// `concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures")`, so the + /// tests do not depend on the working directory. + pub const fn new(dir: &'static str) -> Self { + Self { dir } + } + + /// The path of the fixture file `name`. + pub fn path(&self, name: &str) -> String { + format!("{}/{name}", self.dir) + } + + /// Parse the fixture file `name`. Panics with the path when the file + /// cannot be read or parsed. + pub fn load(&self, name: &str) -> T { + let path = self.path(name); + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); + serde_json::from_str(&text).unwrap_or_else(|e| panic!("{path}: {e}")) + } + + /// A [`Replay`] of the sequence of exchanges in the file `name`. + pub fn replay(&self, name: &str) -> Replay { + Replay::new().expect_all(self.load(name)) + } +} + +/// A body as JSON, the form fixtures keep it in: no bytes is `null`, JSON +/// is its value, and any other bytes are a JSON string of their text. +pub fn body_json(body: &[u8]) -> Value { + if body.is_empty() { + Value::Null + } else { + serde_json::from_slice(body) + .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(body).into())) + } +} + +/// A transport that checks each request against a recorded one and answers +/// with the recorded response. It panics on a request the fixtures do not +/// expect, and keeps the waits the client asks for instead of waiting. +#[derive(Debug, Default)] +pub struct Replay { + exchanges: Mutex, VendorResponse)>>, + sleeps: Mutex>, +} + +impl Replay { + /// A replay that expects no requests. + pub fn new() -> Self { + Self::default() + } + + /// Expect `fixture`'s request and answer with its response. + pub fn expect(self, fixture: Fixture) -> Self { + let response = fixture.response.to_response(); + self.exchanges + .lock() + .unwrap() + .push_back((Some(fixture.request), response)); + self + } + + /// Expect each exchange of a recorded sequence, in order. + pub fn expect_all(mut self, fixtures: Vec) -> Self { + for fixture in fixtures { + self = self.expect(fixture); + } + self + } + + /// Answer the next request, whatever it is, with `response`. + pub fn answer(self, response: VendorResponse) -> Self { + self.exchanges.lock().unwrap().push_back((None, response)); + self + } + + /// Whether every expected request was sent. + pub fn is_done(&self) -> bool { + self.exchanges.lock().unwrap().is_empty() + } + + /// The waits the client asked for, in order. + pub fn sleeps(&self) -> Vec { + self.sleeps.lock().unwrap().clone() + } +} + +impl VendorTransport for Replay { + async fn send(&self, request: VendorRequest) -> Result { + let (expected, response) = self + .exchanges + .lock() + .unwrap() + .pop_front() + .expect("a request the fixtures do not expect"); + if let Some(expected) = expected { + let actual = json!({ + "method": request.method, + "url": request.url, + "headers": request.headers, + "body": body_json(&request.body), + }); + let wanted = json!({ + "method": expected.method, + "url": expected.url, + "headers": expected.headers, + "body": expected.body, + }); + assert_eq!( + actual, + wanted, + "the request differs from the fixture; actual:\n{}", + serde_json::to_string_pretty(&actual).unwrap() + ); + } + Ok(response) + } + + async fn sleep(&self, duration: Duration) { + self.sleeps.lock().unwrap().push(duration); + } +} + +/// A transport that sends through `inner` and keeps every exchange, to +/// write fixtures. It keeps only the `content-type` header of a response. +#[derive(Debug)] +pub struct Recording { + /// The transport that sends. + pub inner: T, + /// The exchanges so far. + pub exchanges: Mutex>, +} + +impl Recording { + /// A recording over `inner`, with no exchanges. + pub fn new(inner: T) -> Self { + Self { + inner, + exchanges: Mutex::new(Vec::new()), + } + } + + /// The exchanges so far, and forget them. + pub fn take(&self) -> Vec { + std::mem::take(&mut *self.exchanges.lock().unwrap()) + } +} + +impl VendorTransport for Recording { + async fn send(&self, request: VendorRequest) -> Result { + let recorded = FixtureRequest { + method: request.method.clone(), + url: request.url.clone(), + headers: request.headers.clone(), + body: body_json(&request.body), + }; + let response = self.inner.send(request).await?; + let headers = response + .header_value("content-type") + .map(|value| vec![("content-type".to_owned(), value.to_owned())]) + .unwrap_or_default(); + self.exchanges.lock().unwrap().push(Fixture { + request: recorded, + response: FixtureResponse { + status: response.status, + headers, + body: body_json(&response.body), + }, + }); + Ok(response) + } + + async fn sleep(&self, duration: Duration) { + self.inner.sleep(duration).await; + } +} From 2d44caa22a892c6789f403186aa2a06dd2894e17 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Fri, 6 Nov 2026 21:59:36 +1100 Subject: [PATCH 2/2] ci: share one job between the two live key vendor workflows integration-gcp-kms.yml and integration-azure-key-vault.yml had the same job: a secret pre-flight, checkout, mise, the Rust cache and mise run test:live:. Only the secrets were different. Move that job to the reusable workflow _test-kms-live.yml. Each caller keeps its triggers, its permissions, its concurrency group, and the job id "live" with its name, its opt-in on the repository variable and its fork-PR condition. So both stay inert until STACK_KMS_GCP_LIVE_KEY or STACK_KMS_AZURE_LIVE_KEY is set, and the scans in workflow-dispatch-job-conditions.test.mjs see the same jobs. The secret names and the pre-flight messages do not change. The Google key still goes to a file only the runner user can read, and the Azure secrets still reach only the test step. Both path filters now include the shared file. The reusable workflow runs mise run test:live:${{ inputs.crate }}. crates-ci.test.mjs read only the run lines of a workflow's own steps, so it saw no caller for the two live tasks. It now reads the steps of a reusable workflow in this repository through each job that calls it, with the caller's inputs put in. A task counts for the caller, so removing a caller still orphans its task. --- .github/workflows/_test-kms-live.yml | 149 ++++++++++++++++++ .../workflows/integration-azure-key-vault.yml | 76 +++------ .github/workflows/integration-gcp-kms.yml | 73 +++------ AGENTS.md | 2 +- scripts/__tests__/crates-ci.test.mjs | 32 +++- 5 files changed, 220 insertions(+), 112 deletions(-) create mode 100644 .github/workflows/_test-kms-live.yml diff --git a/.github/workflows/_test-kms-live.yml b/.github/workflows/_test-kms-live.yml new file mode 100644 index 000000000..e74a1aa98 --- /dev/null +++ b/.github/workflows/_test-kms-live.yml @@ -0,0 +1,149 @@ +name: Key vendor backend live test + +# The one job behind the opt-in live workflows of the key vendor backends +# that have no emulator: integration-gcp-kms.yml (Google Cloud KMS) and +# integration-azure-key-vault.yml (Azure Key Vault). Each caller keeps its +# triggers, its opt-in and fork-PR condition on the calling job, and the +# names of its repository variables and secrets. This file holds what was +# the same in both: the secret pre-flight, the toolchain, and the +# `mise run test:live:` step. +# +# A caller gives: +# +# * `crate`: the crate. The test task is `test:live:`. +# * `name`: the name of the test step. +# * `live-key-variable`: the repository variable that turns the caller on, +# for the pre-flight message. +# * `required-secrets`: one line for each secret the test needs: +# ` `. The pre-flight fails +# the job when one of them is empty: a credentialed job that skips for +# want of a secret reads exactly like one that passes. +# * `live-env`: `NAME=value` lines (repository variables, not secrets) +# that the test reads. +# * the secrets, by the names below. `credentials-json` goes to a file +# only this user can read, outside the checkout, and its path goes in +# the variable `credentials-file-env` names. The `azure-*` secrets reach +# only the test step's environment. + +on: + workflow_call: + inputs: + crate: + type: string + required: true + name: + type: string + required: true + live-key-variable: + type: string + required: true + required-secrets: + type: string + required: true + live-env: + type: string + required: true + credentials-file-env: + type: string + default: '' + secrets: + credentials-json: + required: false + azure-tenant-id: + required: false + azure-client-id: + required: false + azure-client-secret: + required: false + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + live: + runs-on: blacksmith-16vcpu-ubuntu-2204 + timeout-minutes: 30 + env: + MISE_ENV: test + steps: + # Fast pre-flight: fail in seconds if a secret was rotated or cleared, + # before the toolchain install and the compile. + - name: Require the secrets + env: + LIVE_KEY_VARIABLE: ${{ inputs.live-key-variable }} + REQUIRED_SECRETS: ${{ inputs.required-secrets }} + credentials_json: ${{ secrets.credentials-json }} + azure_tenant_id: ${{ secrets.azure-tenant-id }} + azure_client_id: ${{ secrets.azure-client-id }} + azure_client_secret: ${{ secrets.azure-client-secret }} + run: | + missing=0 + while read -r input secret; do + [ -n "$input" ] || continue + name="${input//-/_}" + if [ -z "${!name}" ]; then + echo "::error::$LIVE_KEY_VARIABLE is set but the secret $secret is empty" + missing=1 + fi + done <<< "$REQUIRED_SECRETS" + exit "$missing" + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: . + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2204 + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + + # Repository variables, not secrets. Each line must be NAME=value. + - name: Set the live key variables + env: + LIVE_ENV: ${{ inputs.live-env }} + run: | + while IFS= read -r line; do + [ -n "$line" ] || continue + if ! [[ "$line" =~ ^[A-Z][A-Z0-9_]*= ]]; then + echo "::error::live-env line is not NAME=value" + exit 1 + fi + echo "$line" >> "$GITHUB_ENV" + done <<< "$LIVE_ENV" + + - name: Write the credentials file + if: inputs.credentials-file-env != '' + env: + CREDENTIALS_JSON: ${{ secrets.credentials-json }} + CREDENTIALS_FILE_ENV: ${{ inputs.credentials-file-env }} + CRATE: ${{ inputs.crate }} + run: | + umask 077 + printf '%s' "$CREDENTIALS_JSON" > "$RUNNER_TEMP/$CRATE-live.json" + echo "$CREDENTIALS_FILE_ENV=$RUNNER_TEMP/$CRATE-live.json" >> "$GITHUB_ENV" + + - name: ${{ inputs.name }} + env: + AZURE_TENANT_ID: ${{ secrets.azure-tenant-id }} + AZURE_CLIENT_ID: ${{ secrets.azure-client-id }} + AZURE_CLIENT_SECRET: ${{ secrets.azure-client-secret }} + # The input is a constant in a caller in this repository. It is + # written into the command, not passed through the environment, so + # scripts/__tests__/crates-ci.test.mjs can see which task runs. + run: mise run test:live:${{ inputs.crate }} + + - name: Remove the credentials file + if: always() && inputs.credentials-file-env != '' + env: + CRATE: ${{ inputs.crate }} + run: rm -f "$RUNNER_TEMP/$CRATE-live.json" diff --git a/.github/workflows/integration-azure-key-vault.yml b/.github/workflows/integration-azure-key-vault.yml index 80868905d..5a0222354 100644 --- a/.github/workflows/integration-azure-key-vault.yml +++ b/.github/workflows/integration-azure-key-vault.yml @@ -25,8 +25,12 @@ name: Integration — stack-kms-azure (Azure Key Vault) # STACK_KMS_AZURE_CLIENT_SECRET. # # Once the variable is set, a missing or empty secret FAILS the job (the -# pre-flight below): a credentialed job that skips for want of a secret reads -# exactly like one that passes. +# pre-flight in _test-kms-live.yml): a credentialed job that skips for want +# of a secret reads exactly like one that passes. +# +# The job itself is _test-kms-live.yml, shared with integration-gcp-kms.yml. +# This file keeps the triggers, the opt-in, the fork-PR condition and the +# names of the variables and the secrets. # # Fork PRs have no secrets, so the job skips them; every other event runs. # Enforced by scripts/__tests__/workflow-dispatch-job-conditions.test.mjs. @@ -40,6 +44,7 @@ on: - packages/stack-kms/** - Cargo.lock - .github/workflows/integration-azure-key-vault.yml + - .github/workflows/_test-kms-live.yml pull_request: # Repeated verbatim: GitHub Actions does not support YAML anchors/aliases. paths: @@ -48,15 +53,12 @@ on: - packages/stack-kms/** - Cargo.lock - .github/workflows/integration-azure-key-vault.yml + - .github/workflows/_test-kms-live.yml workflow_dispatch: {} permissions: contents: read -defaults: - run: - shell: bash - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -64,53 +66,23 @@ concurrency: jobs: live: name: stack-kms-azure against Azure Key Vault - runs-on: blacksmith-16vcpu-ubuntu-2204 - timeout-minutes: 30 if: >- vars.STACK_KMS_AZURE_LIVE_KEY != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - env: - MISE_ENV: test - STACK_KMS_AZURE_LIVE_KEY: ${{ vars.STACK_KMS_AZURE_LIVE_KEY }} - STACK_KMS_AZURE_LIVE_HSM_KEY: ${{ vars.STACK_KMS_AZURE_LIVE_HSM_KEY }} - steps: - # Fast pre-flight: fail in seconds if a secret was rotated or cleared, - # before the toolchain install and the compile. - - name: Require the Azure service principal - env: - TENANT_ID: ${{ secrets.STACK_KMS_AZURE_TENANT_ID }} - CLIENT_ID: ${{ secrets.STACK_KMS_AZURE_CLIENT_ID }} - CLIENT_SECRET: ${{ secrets.STACK_KMS_AZURE_CLIENT_SECRET }} - run: | - missing=0 - for name in TENANT_ID CLIENT_ID CLIENT_SECRET; do - if [ -z "${!name}" ]; then - echo "::error::STACK_KMS_AZURE_LIVE_KEY is set but the secret STACK_KMS_AZURE_${name} is empty" - missing=1 - fi - done - exit "$missing" - - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 - with: - version: 2026.4.0 - install: true - working_directory: . - cache: true - # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs - cache_key_prefix: mise-v1-ubuntu-2204 - - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - - # The secrets reach only this step's environment. - - name: stack-kms-azure against Azure Key Vault - env: - AZURE_TENANT_ID: ${{ secrets.STACK_KMS_AZURE_TENANT_ID }} - AZURE_CLIENT_ID: ${{ secrets.STACK_KMS_AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.STACK_KMS_AZURE_CLIENT_SECRET }} - run: mise run test:live:stack-kms-azure + uses: ./.github/workflows/_test-kms-live.yml + with: + crate: stack-kms-azure + name: stack-kms-azure against Azure Key Vault + live-key-variable: STACK_KMS_AZURE_LIVE_KEY + required-secrets: | + azure-tenant-id STACK_KMS_AZURE_TENANT_ID + azure-client-id STACK_KMS_AZURE_CLIENT_ID + azure-client-secret STACK_KMS_AZURE_CLIENT_SECRET + live-env: | + STACK_KMS_AZURE_LIVE_KEY=${{ vars.STACK_KMS_AZURE_LIVE_KEY }} + STACK_KMS_AZURE_LIVE_HSM_KEY=${{ vars.STACK_KMS_AZURE_LIVE_HSM_KEY }} + secrets: + azure-tenant-id: ${{ secrets.STACK_KMS_AZURE_TENANT_ID }} + azure-client-id: ${{ secrets.STACK_KMS_AZURE_CLIENT_ID }} + azure-client-secret: ${{ secrets.STACK_KMS_AZURE_CLIENT_SECRET }} diff --git a/.github/workflows/integration-gcp-kms.yml b/.github/workflows/integration-gcp-kms.yml index a216c0c84..45d1aa5b1 100644 --- a/.github/workflows/integration-gcp-kms.yml +++ b/.github/workflows/integration-gcp-kms.yml @@ -18,8 +18,12 @@ name: Integration — stack-kms-gcp (Google Cloud KMS) # repository secret STACK_KMS_GCP_SERVICE_ACCOUNT_KEY to the JSON key. # # Once the variable is set, a missing or empty secret FAILS the job (the -# pre-flight below): a credentialed job that skips for want of a secret reads -# exactly like one that passes. +# pre-flight in _test-kms-live.yml): a credentialed job that skips for want +# of a secret reads exactly like one that passes. +# +# The job itself is _test-kms-live.yml, shared with +# integration-azure-key-vault.yml. This file keeps the triggers, the opt-in, +# the fork-PR condition and the names of the variable and the secret. # # Fork PRs have no secrets, so the job skips them; every other event runs. # Enforced by scripts/__tests__/workflow-dispatch-job-conditions.test.mjs. @@ -33,6 +37,7 @@ on: - packages/stack-kms/** - Cargo.lock - .github/workflows/integration-gcp-kms.yml + - .github/workflows/_test-kms-live.yml pull_request: # Repeated verbatim: GitHub Actions does not support YAML anchors/aliases. paths: @@ -41,15 +46,12 @@ on: - packages/stack-kms/** - Cargo.lock - .github/workflows/integration-gcp-kms.yml + - .github/workflows/_test-kms-live.yml workflow_dispatch: {} permissions: contents: read -defaults: - run: - shell: bash - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -57,55 +59,18 @@ concurrency: jobs: live: name: stack-kms-gcp against Cloud KMS - runs-on: blacksmith-16vcpu-ubuntu-2204 - timeout-minutes: 30 if: >- vars.STACK_KMS_GCP_LIVE_KEY != '' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) - env: - MISE_ENV: test - STACK_KMS_GCP_LIVE_KEY: ${{ vars.STACK_KMS_GCP_LIVE_KEY }} - steps: - # Fast pre-flight: fail in seconds if the secret was rotated or - # cleared, before the toolchain install and the compile. - - name: Require the Cloud KMS service account key - env: - SERVICE_ACCOUNT_KEY: ${{ secrets.STACK_KMS_GCP_SERVICE_ACCOUNT_KEY }} - run: | - if [ -z "$SERVICE_ACCOUNT_KEY" ]; then - echo "::error::STACK_KMS_GCP_LIVE_KEY is set but the secret STACK_KMS_GCP_SERVICE_ACCOUNT_KEY is empty" - exit 1 - fi - - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 - with: - version: 2026.4.0 - install: true - working_directory: . - cache: true - # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs - cache_key_prefix: mise-v1-ubuntu-2204 - - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - - # The key goes to a file only this user can read, outside the - # checkout, and is removed after the test. - - name: Write the service account key - env: - SERVICE_ACCOUNT_KEY: ${{ secrets.STACK_KMS_GCP_SERVICE_ACCOUNT_KEY }} - run: | - umask 077 - printf '%s' "$SERVICE_ACCOUNT_KEY" > "$RUNNER_TEMP/stack-kms-gcp-live.json" - echo "GOOGLE_APPLICATION_CREDENTIALS=$RUNNER_TEMP/stack-kms-gcp-live.json" >> "$GITHUB_ENV" - - - name: stack-kms-gcp against Cloud KMS - run: mise run test:live:stack-kms-gcp - - - name: Remove the service account key - if: always() - run: rm -f "$RUNNER_TEMP/stack-kms-gcp-live.json" + uses: ./.github/workflows/_test-kms-live.yml + with: + crate: stack-kms-gcp + name: stack-kms-gcp against Cloud KMS + live-key-variable: STACK_KMS_GCP_LIVE_KEY + required-secrets: credentials-json STACK_KMS_GCP_SERVICE_ACCOUNT_KEY + live-env: STACK_KMS_GCP_LIVE_KEY=${{ vars.STACK_KMS_GCP_LIVE_KEY }} + # The test reads the service account key file this names. + credentials-file-env: GOOGLE_APPLICATION_CREDENTIALS + secrets: + credentials-json: ${{ secrets.STACK_KMS_GCP_SERVICE_ACCOUNT_KEY }} diff --git a/AGENTS.md b/AGENTS.md index 9a78a3c4b..38b138de5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,7 +91,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l `cipherstash/encrypt-query-language` repository. Old upstream issue and PR links are provenance only. - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". -- `packages/stack-kms-vendor`, `packages/stack-kms-aws`, `packages/stack-kms-vault`, `packages/stack-kms-gcp`, `packages/stack-kms-azure`: The key vendor backends (ADR-0009 to ADR-0011 in `packages/stack-encrypt/docs/adr/`). `stack-kms-vendor` is the vendor seam (`KeyVendorBackend`), keyset records, both key modes, the static key backend and the `VendorTransport` HTTP seam (feature `http`: the native reqwest transport; the WASI guest's transport is `stack-guest-abi`'s `vendor` feature). Its `test-support` feature holds the test code every vendor crate shares: the contract suite (`contract`), `FakeVendor`, and the recorded-fixture harness (`replay`: fixture types, `Replay`, `Recording`); a vendor crate's `tests/common/mod.rs` holds only its own emulator, keys and constants. `stack-kms-aws` is the AWS KMS backend: a hand-written JSON client with SigV4 signing in-crate and the AWS credential providers; its LocalStack test runs in `tests-crates.yml`'s `localstack` job. `stack-kms-vault` is the HashiCorp Vault Transit backend: a hand-written Transit client with `batch_input` calls and token, AppRole and Kubernetes auth; its `vault server -dev` test runs in `tests-crates.yml`'s `vault` job. `stack-kms-gcp` is the Google Cloud KMS backend: a hand-written Cloud KMS REST client with CRC32C checks, service account (RS256 JWT, the pure-Rust `rsa` crate, which has the open advisory RUSTSEC-2023-0071) and metadata server credentials; the service account login is the default feature `service-account`, and the Go guest and `@cipherstash/auth` build the crate without it, so they hold no RSA code; Google has no Cloud KMS emulator, so its live test runs only in the opt-in, credentialed `integration-gcp-kms.yml`, which is inert until the `STACK_KMS_GCP_LIVE_KEY` repository variable is set. `stack-kms-azure` is the Azure Key Vault and Managed HSM backend: a hand-written Key Vault REST client (`wrapkey`/`unwrapkey`, RSA-OAEP-256 or A256KW, the associated data bound by `stack_kms_vendor::bound` because Key Vault has none) with Entra ID client secret, workload identity and managed identity credentials, and no RSA code of its own; Microsoft has no Key Vault emulator, so its live test runs only in the opt-in, credentialed `integration-azure-key-vault.yml`, which is inert until the `STACK_KMS_AZURE_LIVE_KEY` repository variable is set. All five are root workspace members, publishable in the `stack-kms-vendor` version group, and staged in `release-plz.toml` (`release = false`) until their first version is published by hand. The `stack-kms-vendor` crate docs hold the capability matrix of every key backend (key lifetime, vendor calls, what the vendor sees and can refuse, credentials, rotation, known limits); change it with the code it describes, and keep the short matrix in the Go `encrypt` package docs in step. The `@cipherstash/auth` binding compiles all five in for `stash keyset`, so a change under their `src/` needs an `@cipherstash/auth` changeset too. TypeScript (`@cipherstash/stack`) cannot use a key vendor backend yet. +- `packages/stack-kms-vendor`, `packages/stack-kms-aws`, `packages/stack-kms-vault`, `packages/stack-kms-gcp`, `packages/stack-kms-azure`: The key vendor backends (ADR-0009 to ADR-0011 in `packages/stack-encrypt/docs/adr/`). `stack-kms-vendor` is the vendor seam (`KeyVendorBackend`), keyset records, both key modes, the static key backend and the `VendorTransport` HTTP seam (feature `http`: the native reqwest transport; the WASI guest's transport is `stack-guest-abi`'s `vendor` feature). Its `test-support` feature holds the test code every vendor crate shares: the contract suite (`contract`), `FakeVendor`, and the recorded-fixture harness (`replay`: fixture types, `Replay`, `Recording`); a vendor crate's `tests/common/mod.rs` holds only its own emulator, keys and constants. `stack-kms-aws` is the AWS KMS backend: a hand-written JSON client with SigV4 signing in-crate and the AWS credential providers; its LocalStack test runs in `tests-crates.yml`'s `localstack` job. `stack-kms-vault` is the HashiCorp Vault Transit backend: a hand-written Transit client with `batch_input` calls and token, AppRole and Kubernetes auth; its `vault server -dev` test runs in `tests-crates.yml`'s `vault` job. `stack-kms-gcp` is the Google Cloud KMS backend: a hand-written Cloud KMS REST client with CRC32C checks, service account (RS256 JWT, the pure-Rust `rsa` crate, which has the open advisory RUSTSEC-2023-0071) and metadata server credentials; the service account login is the default feature `service-account`, and the Go guest and `@cipherstash/auth` build the crate without it, so they hold no RSA code; Google has no Cloud KMS emulator, so its live test runs only in the opt-in, credentialed `integration-gcp-kms.yml`, which is inert until the `STACK_KMS_GCP_LIVE_KEY` repository variable is set. `stack-kms-azure` is the Azure Key Vault and Managed HSM backend: a hand-written Key Vault REST client (`wrapkey`/`unwrapkey`, RSA-OAEP-256 or A256KW, the associated data bound by `stack_kms_vendor::bound` because Key Vault has none) with Entra ID client secret, workload identity and managed identity credentials, and no RSA code of its own; Microsoft has no Key Vault emulator, so its live test runs only in the opt-in, credentialed `integration-azure-key-vault.yml`, which is inert until the `STACK_KMS_AZURE_LIVE_KEY` repository variable is set. Both live workflows call one job, `_test-kms-live.yml`; each keeps its triggers, opt-in and fork-PR condition, and its variable and secret names. All five are root workspace members, publishable in the `stack-kms-vendor` version group, and staged in `release-plz.toml` (`release = false`) until their first version is published by hand. The `stack-kms-vendor` crate docs hold the capability matrix of every key backend (key lifetime, vendor calls, what the vendor sees and can refuse, credentials, rotation, known limits); change it with the code it describes, and keep the short matrix in the Go `encrypt` package docs in step. The `@cipherstash/auth` binding compiles all five in for `stash keyset`, so a change under their `src/` needs an `@cipherstash/auth` changeset too. TypeScript (`@cipherstash/stack`) cannot use a key vendor backend yet. - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate and the five key vendor crates included (the binding compiles them in for `keysetCreate` and `keysetRewrap`, which `stash keyset` calls), needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) diff --git a/scripts/__tests__/crates-ci.test.mjs b/scripts/__tests__/crates-ci.test.mjs index 8937f5529..0c19c8f5f 100644 --- a/scripts/__tests__/crates-ci.test.mjs +++ b/scripts/__tests__/crates-ci.test.mjs @@ -193,15 +193,37 @@ function expandMatrix(run, job) { const jobSteps = (job) => (Array.isArray(job?.steps) ? job.steps : []) +/** + * The `run:` bodies of a job. A job that calls a reusable workflow in this + * repository (`uses: ./.github/workflows/_x.yml`) runs that workflow's steps, + * with each `${{ inputs. }}` set to the caller's `with.`. Those + * bodies count for the CALLER: `_test-kms-live.yml` runs + * `mise run test:live:${{ inputs.crate }}`, which names no task until a + * caller says which crate, and deleting a caller must orphan its task. + */ +function jobBodies(job) { + const uses = String(job?.uses ?? '') + if (uses.startsWith(`./${WORKFLOW_DIR}/`)) { + const callee = readWorkflow(uses.slice(2)) + const inputs = job?.with ?? {} + return Object.values(callee?.jobs ?? {}).flatMap((calleeJob) => + jobBodies(calleeJob).map((body) => + body.replace(/\$\{\{\s*inputs\.([\w-]+)\s*\}\}/g, (whole, key) => + key in inputs ? String(inputs[key]) : whole, + ), + ), + ) + } + return jobSteps(job) + .filter((step) => typeof step?.run === 'string') + .flatMap((step) => expandMatrix(step.run, job)) +} + /** Every root workflow, with the `run:` bodies of all its jobs' steps. */ const ROOT_WORKFLOWS = workflowFiles().map((relPath) => { const wf = readWorkflow(relPath) const body = Object.values(wf?.jobs ?? {}) - .flatMap((job) => - jobSteps(job) - .filter((step) => typeof step?.run === 'string') - .flatMap((step) => expandMatrix(step.run, job)), - ) + .flatMap(jobBodies) .join('\n') return { relPath, wf, body } })