feat(go): key vendor backends through the encrypt guest (CIP-5311)

.github/workflows/tests-golang.yml

@@ -41,6 +41,12 @@ on:
       - conformance/stack-encrypt/**
       - packages/stack-encrypt-derive/**
       - packages/stack-guest-abi/**
+      # The encrypt guest links every key vendor backend (CIP-5311).
+      - packages/stack-kms-vendor/**
+      - packages/stack-kms-aws/**
+      - packages/stack-kms-vault/**
+      - packages/stack-kms-gcp/**
+      - packages/stack-kms-azure/**
       # The eql guest build links eql-bindings.
       - packages/eql/crates/eql-bindings/**
       - packages/eql/crates/eql-domains/**
@@ -84,6 +90,12 @@ on:
       - conformance/stack-encrypt/**
       - packages/stack-encrypt-derive/**
       - packages/stack-guest-abi/**
+      # The encrypt guest links every key vendor backend (CIP-5311).
+      - packages/stack-kms-vendor/**
+      - packages/stack-kms-aws/**
+      - packages/stack-kms-vault/**
+      - packages/stack-kms-gcp/**
+      - packages/stack-kms-azure/**
       # The eql guest build links eql-bindings.
       - packages/eql/crates/eql-bindings/**
       - packages/eql/crates/eql-domains/**

AGENTS.md

@@ -759,6 +759,12 @@ exact pin would stop the suite sharing one `cts-common` with it.
   `mutants.yml`, `fuzz.yml`, `miri.yml` and `udeps.yml`.
   `scripts/__tests__/crates-ci.test.mjs` fails when a mise task in the root
   `mise.toml` or the ten crate `tasks.toml` files loses its last CI caller.
+- **The encrypt guest links the key vendor crates.** It builds
+  `stack-kms-vendor` and the four vendor crates in, `stack-kms-gcp`
+  without `service-account`, so the guest holds no RSA code, and sends
+  vendor requests through `stack-guest-abi`'s `vendor` feature. Keep the
+  short key vendor matrix in the Go `encrypt` package docs in step with
+  the `stack-kms-vendor` crate docs.
 - **Auth refresh cancellation soak:** `tests-auth-refresh.yml` runs a short,
   seeded, virtual-time soak on every PR and the long run on auth changes and
   Version Packages PRs. `release-plz.yml` also requires the long run before

languages/golang/encrypt/client.go

@@ -16,11 +16,12 @@ import (
 )
 
 // Client is one wasm instance holding one ZeroKMS client: its key, its
-// default keyset, and the keysets it has loaded since. It is safe for
+// default keyset, and the keysets it has loaded since — and, when it is
+// given one, a key vendor backend with its keyset records. It is safe for
 // concurrent use; the guest runs one call's step at a time, because a wasm
 // instance is single-threaded, but never waits on the network inside one:
-// a ZeroKMS request is performed between steps, with the client free for
-// other callers (see step.go). Close it when done, as with any resource.
+// a ZeroKMS or key vendor request is performed between steps, with the
+// client free for other callers (see step.go). Close it when done, as with any resource.
 //
 // A call whose context ends fails with the context's error, and only that
 // call: one request's cancellation or deadline never closes the client for
@@ -86,17 +87,38 @@ type Client struct {
 // With no options it is a working client: the credentials are
 // [AutoCredentials], and every other setting has a default. Each
 // [ClientOption] changes one.
+//
+// With [WithKeyVendor] and [WithKeysetRecords] the client also holds a key
+// vendor backend. NewClient parses and checks the records but makes no
+// vendor call: a record key is unwrapped on the first use of its keyset.
 func NewClient(ctx context.Context, opts ...ClientOption) (_ *Client, err error) {
 	var cfg clientOptions
 	for _, opt := range opts {
 		opt(&cfg)
 	}
+	// A StaticKey is consumed like the client key: wiped whatever the
+	// outcome, a refused config and a replaced option included.
+	defer func() {
+		wipeVendorKey(cfg.keyVendor)
+		for _, v := range cfg.supersededVendors {
+			wipeVendorKey(v)
+		}
+	}()
 	rt := cfg.transport
 	if rt == nil {
 		rt = http.DefaultTransport
 	}
+	// A key vendor backend is an alternative to ZeroKMS: with one and no
+	// WithCredentials, the client has no ZeroKMS, and no ZeroKMS
+	// credentials are looked for. WithCredentials(AutoCredentials()) gives
+	// both.
+	vendorOnly := cfg.keyVendor != nil && cfg.credentials == nil && len(cfg.superseded) == 0
 	creds := cfg.credentials
-	if creds == nil {
+	switch {
+	case creds != nil:
+	case vendorOnly:
+		creds = noZeroKMS{}
+	default:
 		creds = AutoCredentials()
 	}
 	// Credentials a later WithCredentials replaced are never resolved, but
@@ -114,10 +136,21 @@ func NewClient(ctx context.Context, opts ...ClientOption) (_ *Client, err error)
 	// for. A refused config still consumes an explicit key, as
 	// WithCredentials promises; any other Credentials has not been asked
 	// yet, so holds nothing of this client's.
-	zerokmsURL, err := zerokmsEndpoint(cfg.zerokmsURL)
+	var zerokmsURL string
+	if !vendorOnly {
+		// A client with no ZeroKMS has no ZeroKMS endpoint to pin.
+		zerokmsURL, err = zerokmsEndpoint(cfg.zerokmsURL)
+	}
 	if err == nil && cfg.keysetCacheSize < 0 {
 		err = fmt.Errorf("%w: WithKeysetCacheSize must not be negative", ErrEncoding)
 	}
+	var vendor *vendorSpec
+	if err == nil {
+		vendor, err = keyVendorConfig(cfg.keyVendor, cfg.keysetRecords)
+	}
+	if err == nil {
+		err = checkVendorOnlyOptions(&cfg, vendorOnly)
+	}
 	if explicit, ok := creds.(*explicitCredentials); ok && err == nil && explicit.token == nil {
 		// Knowable from the credentials as they were built: the one place
 		// a missing token source is decided.
@@ -173,6 +206,11 @@ func NewClient(ctx context.Context, opts ...ClientOption) (_ *Client, err error)
 		clientKey:       resolved.ClientKey,
 		zerokmsURL:      zerokmsURL,
 		keysetCacheSize: cfg.keysetCacheSize,
+		vendor:          vendor,
+		keysetRecords:   cfg.keysetRecords,
+		vendorOnly:      vendorOnly,
+		defaultKeyset:   cfg.defaultKeyset,
+		rootKeyTTL:      cfg.rootKeyTTL,
 	})
 	if err != nil {
 		return nil, err
@@ -184,7 +222,7 @@ func NewClient(ctx context.Context, opts ...ClientOption) (_ *Client, err error)
 	// whatever the init's outcome.
 	resolved.ClientKey.Wipe()
 
-	t := &transport{rt: rt, token: resolved.Token}
+	t := &transport{rt: rt, token: resolved.Token, vendor: vendor}
 	inst, err := newInstance(ctx, wasm, t, guest.PolicyFor(cfg.requireLockedMemory))
 	if err != nil {
 		return nil, err
@@ -304,11 +342,23 @@ type initConfig struct {
 	clientKey       *ClientKey
 	zerokmsURL      string
 	keysetCacheSize int
+	// vendor and keysetRecords are the key vendor backend, nil and empty
+	// when the client has none.
+	vendor        *vendorSpec
+	keysetRecords []string
+	// vendorOnly is a client with the key vendor backend and no ZeroKMS:
+	// the config then has no client id, client key or ZeroKMS endpoint.
+	vendorOnly    bool
+	defaultKeyset KeysetSelector
+	rootKeyTTL    *time.Duration
 }
 
 // encodeConfig renders the se_cipher_init object. The result holds the
 // client key; the caller wipes it, and the key it was read from.
 func encodeConfig(cfg initConfig) ([]byte, error) {
+	if cfg.vendorOnly {
+		return vcffi.Marshal(cfg.vendorFields())
+	}
 	if cfg.clientID == "" || cfg.clientKey.IsZero() {
 		return nil, errors.New("encrypt: the credentials' client id and client key are required")
 	}
@@ -330,9 +380,30 @@ func encodeConfig(cfg initConfig) ([]byte, error) {
 	if cfg.keysetCacheSize > 0 {
 		fields = append(fields, vcvalue.Field{Key: "keyset_cache_size", Value: strconv.Itoa(cfg.keysetCacheSize)})
 	}
+	fields = append(fields, cfg.vendorFields()...)
 	return vcffi.Marshal(fields)
 }
 
+// vendorFields renders the key vendor backend's config keys: none without
+// one. The result can hold a StaticKey's key; the caller wipes what it
+// encodes.
+func (cfg initConfig) vendorFields() vcvalue.Object {
+	if cfg.vendor == nil {
+		return nil
+	}
+	fields := cfg.vendor.configFields(cfg.keysetRecords)
+	if cfg.keysetCacheSize > 0 && cfg.vendorOnly {
+		fields = append(fields, vcvalue.Field{Key: "keyset_cache_size", Value: strconv.Itoa(cfg.keysetCacheSize)})
+	}
+	if cfg.defaultKeyset != nil {
+		fields = append(fields, vcvalue.Field{Key: "default_keyset", Value: cfg.defaultKeyset.selector()})
+	}
+	if cfg.rootKeyTTL != nil {
+		fields = append(fields, vcvalue.Field{Key: "root_key_ttl_ms", Value: strconv.FormatInt(cfg.rootKeyTTL.Milliseconds(), 10)})
+	}
+	return fields
+}
+
 // Close shuts the guest down — the client key and every loaded index key
 // are wiped inside the instance — and releases the runtime and the
 // guest's memory, which is wiped on the way out. Idempotent. Every call
@@ -375,7 +446,9 @@ func (c *Client) Keyset(sel KeysetSelector) *Cipher {
 
 // DefaultKeyset binds the client to its default keyset — the one a ZeroKMS
 // administrator set for this client, which is what naming no keyset
-// resolves to: Rust's StackCipher::default_keyset. Loaded at NewClient, so
+// resolves to: Rust's StackCipher::default_keyset. A client with a key
+// vendor backend and no ZeroKMS has a vendor keyset as its default: its only
+// keyset record, or the one [WithDefaultKeyset] names. Loaded at NewClient, so
 // using it never touches ZeroKMS. There is no way to redefine it from here;
 // which keyset is the default is the server's to say. Any other keyset is
 // [Client.Keyset].
@@ -466,6 +539,8 @@ func (c *Client) drive(ctx context.Context, kms bool, f stepFunc) ([]byte, error
 	step, resumed := f, false
 	var handle uint64
 	var responses []byte
+	var op operation
+	defer op.end()
 	for {
 		out, io, err := c.step(ctx, token, tokenErr, step)
 		if resumed {
@@ -483,6 +558,11 @@ func (c *Client) drive(ctx context.Context, kms bool, f stepFunc) ([]byte, error
 			return nil, ctx.Err()
 		}
 		if io == nil {
+			// The guest reports a failed credential request as a transport
+			// failure and no more; the credential source said why.
+			if err != nil && op.credErr != nil {
+				err = fmt.Errorf("%w (key vendor credential source: %w)", err, op.credErr)
+			}
 			return settle(ctx, out, err)
 		}
 		// A caller that gave up while the guest ran sends no request.
@@ -491,7 +571,7 @@ func (c *Client) drive(ctx context.Context, kms bool, f stepFunc) ([]byte, error
 			c.abort(io.handle)
 			return nil, fmt.Errorf("%w: %w", err, ErrTransport)
 		}
-		responses, err = c.transport.performStep(ctx, io)
+		responses, err = c.transport.performStep(ctx, io, &op)
 		if err != nil {
 			c.abort(io.handle)
 			return nil, err

languages/golang/encrypt/doc.go

@@ -314,10 +314,17 @@
 //
 // The guest imports exactly one host function, a bearer-token fetch, served
 // by the credentials' auth strategy. It never performs a request itself: an
-// operation that needs ZeroKMS pauses and hands the request to the client,
-// which sends it with any [net/http.RoundTripper] and resumes the operation
-// with the response. What crosses per ZeroKMS call is what would
-// cross TLS anyway; derived key material never leaves the guest. Under
+// operation that needs ZeroKMS or the key vendor backend pauses and hands
+// the request to the client, which sends it with any
+// [net/http.RoundTripper] and resumes the operation with the response. A
+// key vendor backend's credential, from the source given with
+// [WithKeyVendor], and a retry's backoff are handed to the client the same
+// way. What crosses per ZeroKMS call is what would cross TLS anyway, and
+// keys derived from a ZeroKMS response never leave the guest: a response
+// alone is not usable. A key vendor backend is different: the host does the
+// TLS, so the host and the RoundTripper see each vendor request and
+// response in plaintext, and those carry unwrapped keys (see [KeyVendor]).
+// Under
 // [AutoCredentials] and [OIDCFederation] the same RoundTripper also carries
 // the authentication requests to CTS, so one scoped to the ZeroKMS host alone
 // is not enough. Under [NewCredentials] those requests go through the store
@@ -340,6 +347,87 @@
 // resolved fail NewClient with [ErrNoCredentials]; credentials that resolve
 // but do not work fail it too, at the one ZeroKMS round trip it makes.
 //
+// # Key vendor backends
+//
+// A key vendor backend is an alternative to ZeroKMS: [AWSKMS],
+// [GoogleCloudKMS], [AzureKeyVault], [VaultTransit] or [StaticKey]. Give it
+// with [WithKeyVendor], and its keyset records (the "stack-keyset:…" texts)
+// with [WithKeysetRecords]:
+//
+//	client, err := encrypt.NewClient(ctx,
+//		encrypt.WithKeyVendor(encrypt.AWSKMS{Region: "us-east-1", Credentials: creds}),
+//		encrypt.WithKeysetRecords(os.Getenv("USERS_KEYSET_RECORD")))
+//
+// With no [WithCredentials] the client has no ZeroKMS and needs no
+// CipherStash credentials. Its default keyset is its only keyset record, or
+// the one [WithDefaultKeyset] names, and a call that needs ZeroKMS fails
+// with [ErrNoZeroKMS]. With WithCredentials too, the client holds both: a
+// keyset in a record goes to the vendor, and every other keyset, the default
+// included, goes to ZeroKMS. Select a vendor keyset by its record's name or
+// id with [Client.Keyset]. A value sealed under one backend never opens
+// under the other.
+//
+// The guest builds and signs every vendor request. The host supplies the
+// credentials, per request: AWS credentials, a Google OAuth access token, an
+// Azure access token for the scope the guest names, or a Vault token. Fill
+// them from the vendor's own Go credential chain through
+// [AWSCredentialsFunc], [AccessTokenFunc] or [OAuth2AccessTokens], which
+// give short-lived credentials. [AWSCredentialsFromEnv] and a fixed Vault
+// token can give long-lived ones, and those cross to the guest on each
+// request too. This package imports no vendor SDK.
+//
+// The host sends the vendor requests and does their TLS. So the host, and
+// the RoundTripper given with [WithTransport], see in plaintext the root key
+// that a wrap request sends and that an unwrap response returns. Unlike a
+// ZeroKMS response, such a key is usable alone. The client wipes the request
+// and response bodies it holds, but not the buffers of net/http, TLS or the
+// RoundTripper. Give WithTransport only a RoundTripper you trust with key
+// material.
+//
+// A keyset record holds the keyset's root key, wrapped by the vendor key.
+// The guest derives each data key and the index key from the root key
+// (HKDF-SHA256), with no vendor call. The database holds no usable key
+// material.
+//
+//	key lifetime        root key: cached in the guest for 5 minutes
+//	                    (WithRootKeyTTL); index key: kept until the
+//	                    keyset leaves the client's keyset cache; data
+//	                    keys: derived per call
+//	vendor calls per    0, or 1 per keyset when the root key is not
+//	  call, per keyset  cached
+//	the vendor can      the root key unwrap: a refusal takes effect when
+//	  refuse            the cached root key expires
+//
+// After the vendor refuses a keyset, the client can no longer encrypt or
+// decrypt its values once the cached root key expires. It can still derive
+// search terms for that keyset while the keyset's index key stays in the
+// client's keyset cache, as it can for a ZeroKMS keyset. The first use of a
+// Vault Transit key or an Azure key costs one more call, a key read.
+//
+// The associated data (the keyset id, the vendor and the key mode) is bound
+// to every wrapped root key. AWS KMS binds it as the encryption context,
+// which CloudTrail logs and key policies can test. Google Cloud KMS binds it
+// as additional authenticated data. Vault Transit and Azure Key Vault wrap
+// a key with a digest of it, which the guest checks after the unwrap; a
+// derived Transit key also gets it as its derivation context. Azure does not
+// see the binding. Nothing outside the process can refuse or audit a use of
+// [StaticKey].
+//
+// A rewrap of a keyset record (stash keyset rewrap) wraps the root key under
+// another vendor key. It does not re-key stored values, and every value
+// still opens. The stack-kms-vendor crate documentation has the full
+// capability matrix: credentials, rotation and known limits of each backend.
+//
+// A vendor keyset refuses ZeroKMS-only features (a lock context, a
+// decryption policy) with an error; it never ignores one. A vendor failure
+// reads as the ZeroKMS one of the same kind: [ErrForbidden] for a refused
+// request or bad credentials, [ErrNotFound] for a missing key, [ErrTransport]
+// for an unreachable vendor, a throttle that outlasted the retries or a
+// failed credential source (whose own error errors.Is also matches), and
+// [ErrAuthentication] for a wrapped root key that does not open. The
+// [*Diagnostic]'s code is "stack_kms::backend", and its "vendor" field names
+// the backend.
+//
 // # Errors
 //
 // The generator and the compiler find a mistake in a declaration, so no call

languages/golang/encrypt/errors.go

@@ -17,7 +17,8 @@ import (
 // two.
 var (
 	// ErrAuthentication is an AEAD open failure: a tampered ciphertext, a
-	// wrong element derivation, or a wrong AAD that reached the AEAD. Against
+	// wrong element derivation, or a wrong AAD that reached the AEAD. A key
+	// vendor backend reports a wrapped key that does not unwrap as this too. Against
 	// ZeroKMS a wrong AAD is usually refused earlier as ErrForbidden, because
 	// every data key is bound to its context.
 	ErrAuthentication = guest.ErrAuthentication
@@ -33,15 +34,21 @@ var (
 	ErrUnauthorized = guest.ErrUnauthorized
 	// ErrForbidden is ZeroKMS refusing the request (HTTP 403): the token is
 	// valid but not permitted, or a data key's bound context did not match
-	// the one presented — the production form of a wrong-AAD open.
+	// the one presented — the production form of a wrong-AAD open. From a
+	// key vendor backend: the vendor refused the request, or the guest
+	// refused the host's credential (expired, or the wrong shape).
 	ErrForbidden = guest.ErrForbidden
 	// ErrNotFound is ZeroKMS reporting a missing resource (HTTP 404): an
-	// unknown keyset name or id, or a data key that does not exist.
+	// unknown keyset name or id, or a data key that does not exist. From a
+	// key vendor backend: the vendor key does not exist.
 	ErrNotFound = guest.ErrNotFound
 	// ErrConflict is ZeroKMS reporting a resource conflict (HTTP 409).
 	ErrConflict = guest.ErrConflict
 	// ErrTransport is a failure to reach ZeroKMS or to read its response:
 	// the transport returned an error, or the endpoint could not be resolved.
+	// From a key vendor backend also: the vendor is unavailable, it went on
+	// throttling past the retries, or the key vendor credential source
+	// failed (its error is attached, so errors.Is matches it too).
 	ErrTransport = guest.ErrTransport
 	// ErrKMS is any other ZeroKMS failure: an unparseable response, invalid
 	// key material, or an unclassified server error.
@@ -58,6 +65,11 @@ var (
 	// [Cipher.Context]. Refused before any key is retrieved; a row stored
 	// under another tenant is a mismatch, never a decrypted value.
 	ErrContextMismatch = guest.ErrContextMismatch
+	// ErrNoZeroKMS is a client that has a key vendor backend and no ZeroKMS
+	// asked for a ZeroKMS keyset: a keyset none of its keyset records names,
+	// or a value a ZeroKMS keyset sealed. Refused before any request. Give
+	// the keyset's record, or ZeroKMS credentials with WithCredentials.
+	ErrNoZeroKMS = guest.ErrNoZeroKMS
 	// ErrMemoryLock is guest memory that could not be locked in RAM (or,
 	// on Linux, excluded from core dumps). NewClient returns it when
 	// WithRequireLockedMemory is given, and so does any later call under