diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 5731b01c48..806d794fe6 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -161,7 +161,7 @@ func (h *Handler) handleConnect(ctx context.Context, md *extproc.RequestMetadata "egress denied: invalid actor certificate") } - if err := h.validateActor(ctx, actorRef); err != nil { + if err := h.validateActor(ctx, actorRef, md.Host); err != nil { return extproc.Result{}, err } @@ -255,9 +255,10 @@ func (h *Handler) lookupPolicy(ctx context.Context, leg string, ref resources.Ac } // validateActor checks the actor certified by the certificate against the control -// plane's current view of that actor: it still exists and it is running. Every -// error it returns is already a client-facing ext_proc denial. -func (h *Handler) validateActor(ctx context.Context, actorRef resources.ActorRef) error { +// plane's current view of that actor: it still exists and it is placed on a +// worker. A refusal is logged with destination. Every error it returns is +// already a client-facing ext_proc denial. +func (h *Handler) validateActor(ctx context.Context, actorRef resources.ActorRef, destination string) error { // Confirm the certified actor still exists. // TODO: this can cause heavy load on ate api server. Change it based on https://github.com/agent-substrate/substrate/issues/592. actor, err := h.apiClient.GetActor(ctx, &ateapipb.GetActorRequest{ @@ -267,12 +268,19 @@ func (h *Handler) validateActor(ctx context.Context, actorRef resources.ActorRef return mapEgressIdentityError(actorRef.Atespace, actorRef.Name, err) } - // The actor performing egress must actually be running. - if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { + // The actor performing egress must be placed on a worker. RESUMING is + // saved together with the worker assignment, so a booting workload can + // reach the network before it answers its wakeup probe. Every other state + // has left its worker or is leaving it. + switch state := actor.GetStatus().GetState(); state { + case ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_RESUMING: + return nil + default: + slog.WarnContext(ctx, "egress denied: actor is not placed on a worker", + slog.Any("actor", actorRef), slog.String("state", state.String()), slog.String("destination", destination)) return extproc.NewReqError(envoy_type.StatusCode_Forbidden, - "egress denied: actor %q/%q is %s, not running", actorRef.Atespace, actorRef.Name, actor.GetStatus().GetState()) + "egress denied: actor %q/%q is %s, not placed on a worker", actorRef.Atespace, actorRef.Name, state) } - return nil } // authenticateActorCertificate turns the mTLS peer certificate Envoy recorded diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 72c8f33cff..a054b74185 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -689,7 +689,7 @@ func TestHandleRequestHeadersAuthorization(t *testing.T) { want envoy_type.StatusCode }{ { - name: "actor is not running", + name: "actor is suspended", actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{ Atespace: testEgressAtespace, @@ -722,6 +722,37 @@ func TestHandleRequestHeadersAuthorization(t *testing.T) { } } +// Egress follows the actor's placement on a worker: a resuming actor is booting +// or restoring there and may fetch what it needs to become ready; every other +// state has left its worker or is leaving it. +func TestHandleRequestHeadersActorState(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + allowed := map[ateapipb.ActorState]bool{ + ateapipb.ActorState_ACTOR_STATE_RESUMING: true, + ateapipb.ActorState_ACTOR_STATE_RUNNING: true, + } + for value := range ateapipb.ActorState_name { + state := ateapipb.ActorState(value) + if state == ateapipb.ActorState_ACTOR_STATE_UNSPECIFIED { + continue + } + t.Run(state.String(), func(t *testing.T) { + actor := runningActor() + actor.Status.State = state + h := egressHandler(ca.roots(), actor, nil) + leaf := ca.issueActorCert(t, "spiffe://substrate-actor.local/ateom-for-actor/foo/bar", actorCertOptions{}) + _, err := h.HandleRequestHeaders(context.Background(), egressMetadata(xfccHeader(leaf))) + if allowed[state] { + if err != nil { + t.Fatalf("HandleRequestHeaders() error = %v, want nil", err) + } + return + } + wantStatus(t, err, envoy_type.StatusCode_Forbidden) + }) + } +} + // An ingress-only router has no actor-identity CA. If an egress CONNECT somehow // reaches it, it must fail closed rather than tunnel unauthenticated traffic. func TestHandleRequestHeadersWithoutConfiguredCA(t *testing.T) { diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index a74f3cc02f..f9d9c0b801 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -617,6 +617,11 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } } }() + // Egress before the first container starts, ingress after the wakeup + // probe (see ateomtunnel.Tunnel.ActivateEgress). + if err := s.tunnel.ActivateEgress(attribution, egress); err != nil { + return nil, err + } // Create and start pause container. The bundle rootfs is composed here — // an overlay of the node's cached image layers plus the bundle's private // upper — because mounting is ateom's job (atelet runs with no @@ -657,7 +662,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload if err := wakeupprobe.WaitAll(ctx, req.GetSpec().GetContainers(), ateomnet.ActorVethIP, wakeupprobe.DialFunc(s.sandboxDialer(req.GetActorUid()))); err != nil { return nil, fmt.Errorf("while waiting for container wakeup probe: %w", err) } - if err := s.tunnel.Activate(ateomstats.ActorAttributionFromRequest(req), s.sandboxDialer(req.GetActorUid()), egress); err != nil { + if err := s.tunnel.ActivateIngress(attribution, s.sandboxDialer(req.GetActorUid())); err != nil { return nil, err } @@ -971,6 +976,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } } }() + // As in RunWorkload: egress before the containers restore, ingress after + // the wakeup probe. + err = s.tunnel.ActivateEgress(attribution, egress) + timing.activate = lap(&tLast) + if err != nil { + return nil, err + } checkpointDir := req.GetActorDirs().GetRestoreDir() if hasDurableVolumes(containers) { @@ -1065,8 +1077,8 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if err != nil { return nil, fmt.Errorf("while waiting for container wakeup probe: %w", err) } - err = s.tunnel.Activate(attribution, s.sandboxDialer(req.GetActorUid()), egress) - timing.activate = lap(&tLast) + err = s.tunnel.ActivateIngress(attribution, s.sandboxDialer(req.GetActorUid())) + timing.activate += lap(&tLast) if err != nil { return nil, err } diff --git a/cmd/ateom-gvisor/phaselog.go b/cmd/ateom-gvisor/phaselog.go index 1509545447..ba6c55f78f 100644 --- a/cmd/ateom-gvisor/phaselog.go +++ b/cmd/ateom-gvisor/phaselog.go @@ -60,7 +60,9 @@ const containerCountKey = "ate.actor.container.count" // restoreTiming is the elapsed time of each RestoreWorkload phase. A phase // left at zero never ran. Under a Data scope pauseRestore and appRestore time -// the cold start that stands in for the restore. +// the cold start that stands in for the restore. activate sums the tunnel's +// egress half, armed before the containers restore, and its ingress half, +// after the wakeup probe. type restoreTiming struct { prep, egressPrepare, netSetup, durableDir time.Duration pauseRootfs, pauseCreate, pauseRestore time.Duration diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 561710101f..0fc9941c0b 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -321,6 +321,11 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } } }() + // As in coldBootActor: egress before the VM restores, ingress after the + // wakeup probe. + if err := s.tunnel.ActivateEgress(p.attribution(), egress); err != nil { + return err + } netDevs, err := ch.SnapshotNetDevices(restoreDir) if err != nil { return fmt.Errorf("while reading snapshot net devices: %w", err) @@ -478,7 +483,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, s.startActorLogForwarding(guestAC, attribution, c.GetName(), c.GetName()) } - if err := s.tunnel.Activate(p.attribution(), s.sandboxDialer(p.actorUID), egress); err != nil { + if err := s.tunnel.ActivateIngress(p.attribution(), s.sandboxDialer(p.actorUID)); err != nil { return err } s.setRunningVM(actorUID, ra) diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 50c8356104..7a97b915ba 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -415,6 +415,12 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } }() + // Egress before the guest boots, ingress after the wakeup probe (see + // ateomtunnel.Tunnel.ActivateEgress). + if err := s.tunnel.ActivateEgress(p.attribution(), egress); err != nil { + return err + } + // Guest sizing + agent kernel params. memMiB, vcpus, kparams := s.guestConfig() @@ -589,7 +595,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re slog.Duration("since_boot", time.Since(tBooted))) ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} - if err := s.tunnel.Activate(p.attribution(), s.sandboxDialer(p.actorUID), egress); err != nil { + if err := s.tunnel.ActivateIngress(p.attribution(), s.sandboxDialer(p.actorUID)); err != nil { return err } s.setRunningVM(actorUID, ra) diff --git a/docs/api-guide.md b/docs/api-guide.md index 75c13b2e86..d7c74d32cf 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -322,6 +322,7 @@ How it behaves: - **When it runs.** On every `ResumeActor` that actually wakes the actor. A `ResumeActor` on an actor that is already `RUNNING` is a no-op and does not probe. - **Block-until-ready semantics.** `ResumeActor` returns successfully only after every container with a `wakeupProbe` has returned HTTP 200. If any container has not done so within `timeoutSeconds` (30s by default), `ResumeActor` fails and the actor moves to `ACTOR_STATE_CRASHED`, as it does for any other error while waking. A crashed actor cannot be resumed directly: `ResumeActor` rejects it with `FAILED_PRECONDITION`. Call `RevertActor` first, which returns it to `ACTOR_STATE_SUSPENDED` with its last snapshot, then resume it again. - **Aggressive polling.** The poll loop is tuned for single-millisecond detection latency: a keep-alive HTTP client with a 1ms interval and 250ms per-request timeout. While the workload is still booting, kernel `RST`s return in microseconds, so the loop spends almost no time blocked; once the listener is up, the next attempt completes on veth-local latency. +- **Egress while waking.** The actor's egress is active before its containers start, so a workload can download what it needs to become ready (a model, a Git repository) before it answers the probe. Its EgressPolicy applies as it does once the actor is `RUNNING`; inbound traffic still waits for the probe. - **Golden snapshot warm-up shortcut.** When **every** container in a template declares `wakeupProbe`, the actor template controller skips its default ~20s "give the workload time to settle" delay before taking the golden snapshot — `ResumeActor` already blocked until the workload reported 200, so the workload is known to be initialized. Templates that omit `wakeupProbe` on any container keep the 20s warm-up as a safety net. - **Snapshot/restore interaction.** The TCP listener is part of the checkpointed RAM, so on resume `wakeupProbe` typically returns 200 on the first attempt, with no observable latency penalty. diff --git a/docs/network-egress.md b/docs/network-egress.md index 741d69674e..1f43eee823 100644 --- a/docs/network-egress.md +++ b/docs/network-egress.md @@ -22,9 +22,9 @@ In the default Kubernetes deployment, the PEP obtains its certificate and privat ## Client Authorization -`atunnel` MUST present an actor-specific client certificate when connecting to the PEP. At actor activation, `atunnel` generates a private key and requests a short-lived certificate from substrate through the node-local atelet. The private key remains in `atunnel`. The certificate identifies the actor by its atespace, name, and UID and is scoped to the `atunnel` purpose. +`atunnel` MUST present an actor-specific client certificate when connecting to the PEP. Before the actor's workload starts, `atunnel` generates a private key and requests a short-lived certificate from substrate through the node-local atelet, and egress is active from then on: a workload may need the network to become ready, so egress does not wait for the wakeup probe the way ingress does. The private key remains in `atunnel`. The certificate identifies the actor by its atespace, name, and UID and is scoped to the `atunnel` purpose. -The PEP MUST require and verify the client certificate before accepting a CONNECT request. It MUST verify the certificate chain and lifetime against the Actor Identity CA, require the client authentication usage, and require exactly one valid `ActorIdentity` extension whose atespace, actor name, actor UID, and purpose are present. The current extension OID is `1.3.6.1.4.1.11129.2.12.2`; it is allocated under Google's enterprise number and will change after the CNCF donation is complete. The purpose MUST be `atunnel`, and the certificate's actor URI SAN MUST identify the same actor as the extension. The PEP MUST then confirm with ate-api-server that the actor still exists, that its current UID matches the certificate, and that it is running. The PEP MUST reject the CONNECT request if any of these checks fail. +The PEP MUST require and verify the client certificate before accepting a CONNECT request. It MUST verify the certificate chain and lifetime against the Actor Identity CA, require the client authentication usage, and require exactly one valid `ActorIdentity` extension whose atespace, actor name, actor UID, and purpose are present. The current extension OID is `1.3.6.1.4.1.11129.2.12.2`; it is allocated under Google's enterprise number and will change after the CNCF donation is complete. The purpose MUST be `atunnel`, and the certificate's actor URI SAN MUST identify the same actor as the extension. The PEP MUST then confirm with ate-api-server that the actor still exists, that its current UID matches the certificate, and that it is placed on a worker: `RESUMING` (booting or restoring there) or `RUNNING`. The PEP MUST reject the CONNECT request if any of these checks fail. ```text CURRENT CONNECT EGRESS PATH (one tunnel per actor TCP connection except destination port 53) diff --git a/internal/ateomtunnel/ateomtunnel.go b/internal/ateomtunnel/ateomtunnel.go index c434d65c3f..826f33c8fa 100644 --- a/internal/ateomtunnel/ateomtunnel.go +++ b/internal/ateomtunnel/ateomtunnel.go @@ -218,12 +218,10 @@ func (t *Tunnel) PrepareEgress(ctx context.Context, actor resources.ActorAttribu return &ActorEgress{client: gatewayClient, certificateSource: certificateSource, expiresAt: expiresAt}, nil } -// Activate starts admitting the actor's traffic. Ingress reaches the actor -// through dial; egress is activated only when it was prepared. -func (t *Tunnel) Activate(actor resources.ActorAttribution, dial atunnel.DialFunc, egress *ActorEgress) error { - if err := t.Ingress.Activate(actor.Ref.Atespace, actor.Ref.Name, actor.UID, dial); err != nil { - return fmt.Errorf("while activating actor ingress: %w", err) - } +// ActivateEgress starts tunneling the actor's outbound traffic. Call it before +// the workload starts, since a workload may need the network to become ready. +// A nil egress is a no-op. +func (t *Tunnel) ActivateEgress(actor resources.ActorAttribution, egress *ActorEgress) error { if egress == nil { return nil } @@ -233,6 +231,15 @@ func (t *Tunnel) Activate(actor resources.ActorAttribution, dial atunnel.DialFun return nil } +// ActivateIngress starts admitting the actor's inbound traffic through dial. +// Call it once the workload is ready to serve. +func (t *Tunnel) ActivateIngress(actor resources.ActorAttribution, dial atunnel.DialFunc) error { + if err := t.Ingress.Activate(actor.Ref.Atespace, actor.Ref.Name, actor.UID, dial); err != nil { + return fmt.Errorf("while activating actor ingress: %w", err) + } + return nil +} + // Deactivate stops admitting the actor's traffic and drains its active streams, // before the actor network is torn down. It attempts both directions even if // one fails. diff --git a/internal/ateomtunnel/ateomtunnel_test.go b/internal/ateomtunnel/ateomtunnel_test.go index 7f9954a31f..2b5be808f8 100644 --- a/internal/ateomtunnel/ateomtunnel_test.go +++ b/internal/ateomtunnel/ateomtunnel_test.go @@ -275,8 +275,11 @@ func TestActivateAndDeactivate(t *testing.T) { actor := resources.ActorAttribution{Ref: resources.ActorRef{Atespace: "space", Name: "actor"}, UID: "uid"} dial := func(context.Context, string, string) (net.Conn, error) { return nil, net.ErrClosed } - if err := tunnel.Activate(actor, dial, nil); err != nil { - t.Fatalf("Activate without egress: %v", err) + if err := tunnel.ActivateEgress(actor, nil); err != nil { + t.Fatalf("ActivateEgress without a gateway: %v", err) + } + if err := tunnel.ActivateIngress(actor, dial); err != nil { + t.Fatalf("ActivateIngress: %v", err) } if err := tunnel.Deactivate(ctx, actor); err != nil { t.Fatalf("Deactivate: %v", err) @@ -286,8 +289,8 @@ func TestActivateAndDeactivate(t *testing.T) { t.Fatalf("second Deactivate: %v", err) } - err := tunnel.Activate(resources.ActorAttribution{UID: "uid"}, dial, nil) + err := tunnel.ActivateIngress(resources.ActorAttribution{UID: "uid"}, dial) if err == nil || !strings.Contains(err.Error(), "while activating actor ingress") { - t.Fatalf("Activate with no actor reference = %v, want an ingress error", err) + t.Fatalf("ActivateIngress with no actor reference = %v, want an ingress error", err) } } diff --git a/internal/atunnel/egress.go b/internal/atunnel/egress.go index 511bba5234..8ee9437563 100644 --- a/internal/atunnel/egress.go +++ b/internal/atunnel/egress.go @@ -274,10 +274,21 @@ func (e *Egress) handle(downstream net.Conn, active *egressActivation) { _ = downstream.Close() return } + if active.dialer == nil { + e.mu.Unlock() + // From inside the sandbox this is a connection that opens and closes at + // once, indistinguishable from a network failure. + slog.WarnContext(active.ctx, "atunnel dropped an egress connection: actor egress is not active", + slog.String("peer", downstream.RemoteAddr().String())) + _ = downstream.Close() + return + } if time.Now().Compare(active.expiresAt) >= 0 { // Expiry blocks only new tunnels. Connections admitted with a valid // certificate have completed mTLS and are allowed to drain normally. e.mu.Unlock() + slog.WarnContext(active.ctx, "atunnel dropped an egress connection: actor certificate expired", + slog.String("peer", downstream.RemoteAddr().String())) _ = downstream.Close() return } diff --git a/internal/e2e/atenet_dataplane.go b/internal/e2e/atenet_dataplane.go index 25dc80ce3e..b000168b57 100644 --- a/internal/e2e/atenet_dataplane.go +++ b/internal/e2e/atenet_dataplane.go @@ -37,6 +37,7 @@ type AtenetDataplane interface { IsEgressPolicyDenied(status int, body string) bool PlatformMetricPrefixes([]string) []string RouteDurationSeen(context.Context, string) (bool, error) + SupportsEgressWhileResuming() bool } // CurrentAtenetDataplane returns the implementation selected for this test @@ -71,6 +72,8 @@ func (envoyAtenetDataplane) IsEgressPolicyDenied(status int, body string) bool { (status == http.StatusBadGateway && strings.Contains(body, "request failed")) } +func (envoyAtenetDataplane) SupportsEgressWhileResuming() bool { return true } + func (envoyAtenetDataplane) PlatformMetricPrefixes(prefixes []string) []string { return prefixes } func (envoyAtenetDataplane) RouteDurationSeen(_ context.Context, collectorScrape string) (bool, error) { @@ -93,6 +96,9 @@ func (agentGatewayAtenetDataplane) IsEgressPolicyDenied(status int, body string) return status == http.StatusForbidden && strings.Contains(body, "actor egress policy denied") } +// AgentGateway's substrate egress actor resolution admits RUNNING actors only. +func (agentGatewayAtenetDataplane) SupportsEgressWhileResuming() bool { return false } + func (agentGatewayAtenetDataplane) PlatformMetricPrefixes(prefixes []string) []string { filtered := make([]string, 0, len(prefixes)) for _, prefix := range prefixes { diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index f8d61e461d..fe85f59a23 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -95,6 +96,45 @@ func decodeSubstrateTemplates(t *testing.T, rendered []byte) []*ateapipb.ActorTe return templates } +// FixtureOption adjusts what DeploySubstrateFixture installs. +type FixtureOption func(*fixtureConfig) + +type fixtureConfig struct { + goldenEgress []*ateapipb.EgressRule +} + +// WithGoldenEgressPolicy gives each template's golden actor an EgressPolicy +// with rules, for a fixture whose workload reaches the network before it is +// ready. The gateway denies an actor with no policy, and the golden actor +// exists only once its template does, so the policy lands after the golden +// boot has started: the workload must retry what it fetches. +func WithGoldenEgressPolicy(rules ...*ateapipb.EgressRule) FixtureOption { + return func(c *fixtureConfig) { c.goldenEgress = rules } +} + +// ensureGoldenEgressPolicy waits for tmpl's golden actor to exist and gives it +// an EgressPolicy with rules. Deleting the template deletes the golden actor +// and its policy with it. +func ensureGoldenEgressPolicy(t *testing.T, ctx context.Context, clients *Clients, tmpl *ateapipb.ActorTemplate, rules []*ateapipb.EgressRule) { + t.Helper() + golden := &ateapipb.ObjectRef{Atespace: resources.GoldenActorAtespace, Name: tmpl.GetMetadata().GetUid()} + deadline := time.Now().Add(2 * time.Minute) + for { + _, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{Actor: golden}) + if err == nil { + break + } + if status.Code(err) != codes.NotFound { + t.Fatalf("GetActor for the golden actor of %s/%s: %v", tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName(), err) + } + if time.Now().After(deadline) { + t.Fatalf("golden actor of %s/%s was not created within 2m", tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName()) + } + time.Sleep(500 * time.Millisecond) + } + EnsureEgressPolicy(t, ctx, clients, golden, rules...) +} + // SubstrateFixtureManifests names the two manifest templates a substrate // fixture is built from (both repo-relative, under internal/e2e/fixtures). type SubstrateFixtureManifests struct { @@ -120,9 +160,14 @@ type SubstrateFixtureManifests struct { // // Returns the fixture's atespace (the same string that names the k8s // namespace holding the pool) and the created templates. -func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, manifests SubstrateFixtureManifests, bucket, name string, trustBundle bool) (string, []*ateapipb.ActorTemplate) { +func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, manifests SubstrateFixtureManifests, bucket, name string, trustBundle bool, opts ...FixtureOption) (string, []*ateapipb.ActorTemplate) { t.Helper() + var cfg fixtureConfig + for _, opt := range opts { + opt(&cfg) + } + // The pool manifest also carries the namespace. Its cleanup is registered // first so it runs last (t.Cleanup is LIFO), after the templates that // select the pool's workers are gone. @@ -183,6 +228,12 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, created = append(created, c) } + if cfg.goldenEgress != nil { + for _, tmpl := range created { + ensureGoldenEgressPolicy(t, ctx, clients, tmpl, cfg.goldenEgress) + } + } + for _, tmpl := range created { t.Logf("Waiting for ActorTemplate %s/%s golden snapshot...", atespace, tmpl.GetMetadata().GetName()) WaitForSubstrateTemplateReady(ctx, t, clients, atespace, tmpl.GetMetadata().GetName()) diff --git a/internal/e2e/fixtures/probe/main.go b/internal/e2e/fixtures/probe/main.go index 9db51f30db..815f7212d2 100644 --- a/internal/e2e/fixtures/probe/main.go +++ b/internal/e2e/fixtures/probe/main.go @@ -439,7 +439,75 @@ func writeJSON(w http.ResponseWriter, v any) { } } +// bootFetchURLEnv names a URL the probe must fetch, through the egress gateway +// and with the projected trust bundle, before it listens. A probe that cannot +// fetch it exits and never answers its wakeup probe, so a golden snapshot of +// it proves the actor had egress while it booted. +const bootFetchURLEnv = "PROBE_BOOT_FETCH_URL" + +// bootFetchBudget bounds the boot fetch, inside the template's wakeupProbe +// timeout. +const bootFetchBudget = 45 * time.Second + +// bootFetch GETs url, trusting only trustBundle, until it answers 200 or +// budget runs out. It retries because the gateway caches a missing +// EgressPolicy for its TTL, and a suite can only give the golden actor one +// once the actor exists. +func bootFetch(ctx context.Context, url, trustBundle string, budget time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + b, err := os.ReadFile(trustBundle) + if err != nil { + return fmt.Errorf("reading trust bundle: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(b) { + return fmt.Errorf("no certificates parsed from %s", trustBundle) + } + client := &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}}, + } + defer client.CloseIdleConnections() + for { + lastErr := fetchOnce(ctx, client, url) + if lastErr == nil { + return nil + } + log.Printf("probe: boot fetch of %s: %v", url, lastErr) + select { + case <-ctx.Done(): + return fmt.Errorf("boot fetch of %s: %w", url, lastErr) + case <-time.After(time.Second): + } + } +} + +func fetchOnce(ctx context.Context, client *http.Client, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(res.Body, maxFetchBody)) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("status %d", res.StatusCode) + } + return nil +} + func main() { + if url := os.Getenv(bootFetchURLEnv); url != "" { + if err := bootFetch(context.Background(), url, trustFile, bootFetchBudget); err != nil { + log.Fatalf("probe: %v", err) + } + log.Printf("probe: boot fetch of %s succeeded", url) + } + // Hold the identity file open before serving: every snapshot of this actor // then contains an open guest handle on a system-info file (see // heldIdentity). diff --git a/internal/e2e/fixtures/probe/main_test.go b/internal/e2e/fixtures/probe/main_test.go index f5a44cdd55..44099428bd 100644 --- a/internal/e2e/fixtures/probe/main_test.go +++ b/internal/e2e/fixtures/probe/main_test.go @@ -16,12 +16,17 @@ package main import ( "encoding/json" + "encoding/pem" "net/http" "net/http/httptest" "net/url" + "os" + "path/filepath" "slices" "strings" + "sync/atomic" "testing" + "time" "github.com/google/go-cmp/cmp" ) @@ -295,3 +300,49 @@ func TestCapabilityNamesTable(t *testing.T) { } } } + +// writeServerTrustBundle writes the TLS test server's certificate as the only +// anchor of a trust bundle and returns its path. +func writeServerTrustBundle(t *testing.T, server *httptest.Server) string { + t.Helper() + path := filepath.Join(t.TempDir(), "trust-bundle.pem") + block := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + if err := os.WriteFile(path, block, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +// A golden actor's EgressPolicy can land after its boot started, and the +// gateway caches the missing policy for a while, so the boot fetch retries +// until the origin answers. +func TestBootFetchRetriesUntilOK(t *testing.T) { + var calls atomic.Int32 + origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) < 3 { + w.WriteHeader(http.StatusForbidden) + return + } + w.WriteHeader(http.StatusOK) + })) + defer origin.Close() + + if err := bootFetch(t.Context(), origin.URL, writeServerTrustBundle(t, origin), 30*time.Second); err != nil { + t.Fatalf("bootFetch() error = %v, want nil", err) + } + if got := calls.Load(); got != 3 { + t.Errorf("origin saw %d requests, want 3", got) + } +} + +func TestBootFetchFailsAfterBudget(t *testing.T) { + origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer origin.Close() + + err := bootFetch(t.Context(), origin.URL, writeServerTrustBundle(t, origin), 1500*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "status 403") { + t.Fatalf("bootFetch() error = %v, want the last status", err) + } +} diff --git a/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl b/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl new file mode 100644 index 0000000000..1fa282bd60 --- /dev/null +++ b/internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl @@ -0,0 +1,57 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Boot-fetch variant of the probe template: a protojson-shaped +# ateapipb.ActorTemplate whose probe fetches https://example.com/ through the +# egress gateway before it listens, so its golden snapshot exists only if the +# actor had egress while it booted. Selects the probe pool of probe.yaml.tmpl; +# the suite gives the golden actor its EgressPolicy (see +# e2e.WithGoldenEgressPolicy) and the template the trust bundle. + +metadata: + atespace: ate-e2e-probe${FIXTURE_SUFFIX} + name: probe-bootfetch +workerSelector: + matchLabels: + workload: probe${FIXTURE_SUFFIX} +containers: +- name: probe + image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe + command: ["/ko-app/probe"] + env: + - name: PROBE_BOOT_FETCH_URL + value: https://example.com/ + volumeMounts: + - name: system-info + mountPath: /run/ate + # Longer than the probe's own boot-fetch budget, so a failed fetch is the + # probe's logged error rather than a probe timeout. + wakeupProbe: + httpGet: + path: /healthz + port: 80 + timeoutSeconds: 60 +volumes: +- name: system-info + systemInfo: + dataSources: + - actorMetadata: + items: + - field: ACTOR_METADATA_FIELD_NAME + path: actor-id +${TEMPLATE_TRUST_BUNDLE} +${TEMPLATE_RESOURCES} +${TEMPLATE_SANDBOX_CONFIG} +snapshotConfig: + storageLocation: gs://${BUCKET_NAME}/ate-e2e-probe${FIXTURE_SUFFIX}-bootfetch/ diff --git a/internal/e2e/suites/egressmitm/goldenboot_test.go b/internal/e2e/suites/egressmitm/goldenboot_test.go new file mode 100644 index 0000000000..37907715f9 --- /dev/null +++ b/internal/e2e/suites/egressmitm/goldenboot_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egressmitm + +import ( + "context" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +var bootFetchManifests = e2e.SubstrateFixtureManifests{ + Pool: "internal/e2e/fixtures/probe/probe.yaml.tmpl", + Template: "internal/e2e/fixtures/probe/probe-bootfetch-template.yaml.tmpl", +} + +// TestGoldenBootEgress proves an actor has egress while it boots: the +// probe-bootfetch workload fetches https://example.com/ through the MITM +// gateway before it listens, and exits if it cannot, so its template gets a +// golden snapshot only if the worker tunneled the golden actor's egress +// before the wakeup probe and the gateway admitted the actor while it was +// still resuming. +// +// hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -run TestGoldenBootEgress -v -args --no-color +func TestGoldenBootEgress(t *testing.T) { + if !e2e.CurrentAtenetDataplane().SupportsEgressWhileResuming() { + t.Skip("the agentgateway dataplane admits egress from RUNNING actors only") + } + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + if err != nil { + t.Fatalf("CheckEnv failed: %v", err) + } + ctx := context.Background() + clients := e2e.GetClients() + e2e.EnsureEgressTrustBundle(t, ctx, clients) + + // DeploySubstrateFixture returns once every template has its golden + // snapshot, which is the assertion. + e2e.DeploySubstrateFixture(t, ctx, clients, bootFetchManifests, env["BUCKET_NAME"], "egressmitm-bootfetch", true, + e2e.WithGoldenEgressPolicy(e2e.EgressAllowHTTPS(egressOriginHost))) +}