From beb4a6d222e89d60f4080464ecce29f5625a1dd2 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:42:40 -0700 Subject: [PATCH 1/5] Bump Substrate to 6edd1fcc: string sandbox class, required actor limits Moves the Substrate pin from da35a983 (2026-10-13) to main at 6edd1fcc (2026-10-15). Against that Substrate, env main's ActorTemplate is rejected at registration: - substrate#2586 made SandboxConfig.sandbox_class a string ("gvisor", "microvm") at the same field number. The template now names it with atev1alpha1.SandboxClassGvisor, the constant the WorkerPool already uses. - substrate#2285 requires cpu and memory limits on every ActorTemplate. `ate-env manifest template` writes them, from new --cpu and --memory flags (Kubernetes quantities, default 2 and 4Gi, validated client-side). Templates derived per rootfs image clone the base and inherit them. Compile-only follow-ups: - substrate#2575 renamed ContainerWakeupProbe to WakeupProbe; the field number and JSON key are unchanged. - substrate#2620 deleted ACTOR_STATE_PAUSED and _PAUSING: a paused actor is SUSPENDED with a node-local snapshot. ate-env never pauses, so the two cases leave ActorStatusToEnvStatus; the fake's PauseActor reports SUSPENDED. - substrate#2254 renamed the worker image ateom-gvisor to ateworker-gvisor; the --worker-image help and error text follow, and --ateom-image stays as an alias. --- cmd/ate-env/manifest.go | 53 +++++++++++++++--- cmd/ate-env/manifest_test.go | 54 +++++++++++++++++-- go.mod | 2 +- go.sum | 4 +- internal/apiservice/imagetemplate_test.go | 4 +- internal/apiservice/server.go | 8 +-- internal/apiservice/server_test.go | 6 +-- .../internaltest/fakecontrol/fakecontrol.go | 3 +- 8 files changed, 108 insertions(+), 26 deletions(-) diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index 4894a111..7fe96a7e 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -29,6 +29,7 @@ import ( "google.golang.org/protobuf/proto" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/yaml" @@ -42,6 +43,15 @@ const apiName = "ate-env-api" // one, so a pool declared with it is schedulable on a stock install. const sandboxConfigName = "gvisor-default" +// Default compute limits for each actor. Substrate requires both on every +// ActorTemplate since substrate#2285: the scheduler books them against worker +// capacity. Two CPUs and 4 GiB match the Python client's ResourceLimits +// default; size them for the workload with --cpu and --memory. +const ( + defaultActorCPU = "2" + defaultActorMemory = "4Gi" +) + // manifestConfig holds configuration for generating Kubernetes deployment manifests. type manifestConfig struct { namespace string @@ -56,8 +66,8 @@ type manifestConfig struct { func (c *manifestConfig) resolveImages() error { if c.workerImage == "" { - return errors.New(`--worker-image (or --ateom-image) is required: the worker image is -ateom-gvisor from the Substrate repo and must match the Substrate version + return errors.New(`--worker-image is required: the worker image is +ateworker-gvisor from the Substrate repo and must match the Substrate version deployed on your cluster. From a Substrate checkout at that version run "go run ./cmd/ate-setup publish worker-images": it pushes the images and prints the digest-pinned refs to use. Pin by digest: a registry's :latest @@ -93,6 +103,10 @@ type templateConfig struct { sidecars []string // sidecarReadyz are URLs the guest's /readyz waits for. sidecarReadyz []string + // cpu and memory are the actor's compute limits as Kubernetes quantities. + // Empty means defaultActorCPU and defaultActorMemory. + cpu string + memory string } // runtimeLayer is a parsed --layer value. @@ -148,6 +162,18 @@ binaries default it to the ate-env-guest image published with the release if c.snapshotsBucket == "" { return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") } + for _, l := range []struct{ flag, value string }{{"--cpu", c.cpu}, {"--memory", c.memory}} { + if l.value == "" { + continue + } + q, err := resource.ParseQuantity(l.value) + if err != nil { + return fmt.Errorf("%s %q is not a Kubernetes quantity (e.g. 2, 500m, 4Gi): %w", l.flag, l.value, err) + } + if q.Sign() <= 0 { + return fmt.Errorf("%s must be greater than zero, got %q", l.flag, l.value) + } + } if c.rootfsImage != "" { if _, err := apiservice.ImageDigest(c.rootfsImage); err != nil { return fmt.Errorf("--rootfs-image: %w", err) @@ -203,8 +229,8 @@ the "template" subcommand: ate-env manifest template`, cmd.Flags().StringVar(&mCfg.namespace, "namespace", apiservice.DefaultNamespace, "Kubernetes namespace to deploy into") cmd.Flags().StringVar(&mCfg.template, "template", apiservice.DefaultTemplate, "ActorTemplate name") - cmd.Flags().StringVar(&mCfg.workerImage, "worker-image", "", "digest-pinned worker image for the worker pool, e.g. ateom-gvisor built from the Substrate repo") - cmd.Flags().StringVar(&mCfg.workerImage, "ateom-image", "", "alias for --worker-image") + cmd.Flags().StringVar(&mCfg.workerImage, "worker-image", "", "digest-pinned worker image for the worker pool, e.g. ateworker-gvisor built from the Substrate repo") + cmd.Flags().StringVar(&mCfg.workerImage, "ateom-image", "", "alias for --worker-image, kept from before Substrate renamed ateom to ateworker") cmd.Flags().StringVar(&mCfg.apiImage, "api-image", defaultAPIImage, imageDefaultHelp("digest-pinned ate-env-api image for the API service", defaultAPIImage)) cmd.Flags().Int32Var(&mCfg.apiReplicas, "api-replicas", 1, "number of API service replicas") cmd.Flags().Int32Var(&mCfg.apiPort, "api-port", 7777, "port the ate-env-api service listens on") @@ -250,6 +276,8 @@ It prints YAML to stdout without touching the cluster.`, cmd.Flags().StringArrayVar(&tCfg.layers, "layer", nil, "extra runtime mounted as a read-only image volume, as name=image@sha256:...=/mount/path (repeatable)") cmd.Flags().StringArrayVar(&tCfg.sidecars, "sidecar", nil, "command the guest starts and keeps running next to itself, an absolute binary path plus arguments, e.g. '/opt/opensandbox/execd --port 44772' (repeatable)") cmd.Flags().StringArrayVar(&tCfg.sidecarReadyz, "sidecar-readyz", nil, "URL the actor's readiness waits for, e.g. http://127.0.0.1:44772/ready (repeatable)") + cmd.Flags().StringVar(&tCfg.cpu, "cpu", defaultActorCPU, "CPU limit of each actor, a Kubernetes quantity such as 2 or 500m; Substrate requires one") + cmd.Flags().StringVar(&tCfg.memory, "memory", defaultActorMemory, "memory limit of each actor, a Kubernetes quantity such as 4Gi; Substrate requires one") return cmd } @@ -387,7 +415,7 @@ func buildWorkerPool(cfg manifestConfig) *atev1alpha1.WorkerPool { // the SandboxConfig the workers run; the template names the same // one in sandboxConfig.configName. SandboxClasses: []atev1alpha1.WorkerPoolSandboxClass{{ - Name: atev1alpha1.SandboxClassGvisor, + Name: atev1alpha1.SandboxClassGvisor, // the template's sandboxConfig.sandboxClass is the same string ConfigRef: &atev1alpha1.SandboxConfigReference{Name: sandboxConfigName}, }}, }, @@ -412,7 +440,7 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { Name: "PORT", Value: "80", }}, - WakeupProbe: &ateapipb.ContainerWakeupProbe{ + WakeupProbe: &ateapipb.WakeupProbe{ HttpGet: &ateapipb.HttpGetAction{ Path: "/readyz", Port: 80, @@ -427,10 +455,21 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, }, SandboxConfig: &ateapipb.SandboxConfig{ - SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, + SandboxClass: string(atev1alpha1.SandboxClassGvisor), ConfigName: sandboxConfigName, }, + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ + {Name: "cpu", Quantity: orDefault(cfg.cpu, defaultActorCPU)}, + {Name: "memory", Quantity: orDefault(cfg.memory, defaultActorMemory)}, + }}, + } +} + +func orDefault(v, def string) string { + if v == "" { + return def } + return v } // buildTemplate returns the ActorTemplate for cfg: the guest template from diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 2169d039..5a8ca96e 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -116,11 +116,10 @@ func TestBuildManifests(t *testing.T) { if sc.Name != atev1alpha1.SandboxClassGvisor || sc.ConfigRef == nil || sc.ConfigRef.Name != sandboxConfigName { t.Errorf("workerpool sandboxClasses[0] = %+v, want gvisor with configRef %q", sc, sandboxConfigName) } - // The pool's class is a CRD string and the template's a proto enum: two - // vocabularies for one thing. A pool of the wrong class never hosts the - // template's actors. - tmplClass := strings.ToLower(strings.TrimPrefix( - buildActorTemplate(testTemplateConfig()).GetSandboxConfig().GetSandboxClass().String(), "SANDBOX_CLASS_")) + // The pool's class and the template's are the same string since + // substrate#2586 made the proto field a string. A pool of the wrong class + // never hosts the template's actors. + tmplClass := buildActorTemplate(testTemplateConfig()).GetSandboxConfig().GetSandboxClass() if tmplClass != string(sc.Name) { t.Errorf("workerpool sandbox class %q != template sandbox class %q", sc.Name, tmplClass) } @@ -176,6 +175,44 @@ func TestBuildActorTemplate(t *testing.T) { if probe.GetHttpGet() == nil || probe.GetHttpGet().GetPath() != "/readyz" || probe.GetHttpGet().GetPort() != 80 { t.Errorf("wakeupProbe = %+v, want HTTP GET /readyz on port 80", probe) } + // Substrate rejects an ActorTemplate without both limits (substrate#2285). + if got := limits(template); got["cpu"] != defaultActorCPU || got["memory"] != defaultActorMemory || len(got) != 2 { + t.Errorf("resources.limits = %v, want cpu=%s memory=%s", got, defaultActorCPU, defaultActorMemory) + } + if got := template.GetSandboxConfig().GetSandboxClass(); got != "gvisor" { + t.Errorf("sandbox class = %q, want gvisor (Substrate accepts gvisor or microvm)", got) + } +} + +func limits(tmpl *ateapipb.ActorTemplate) map[string]string { + m := map[string]string{} + for _, l := range tmpl.GetResources().GetLimits() { + m[l.GetName()] = l.GetQuantity() + } + return m +} + +func TestBuildActorTemplateResources(t *testing.T) { + cfg := testTemplateConfig() + cfg.cpu, cfg.memory = "500m", "1Gi" + if got := limits(buildActorTemplate(cfg)); got["cpu"] != "500m" || got["memory"] != "1Gi" { + t.Errorf("resources.limits = %v, want cpu=500m memory=1Gi", got) + } +} + +func TestTemplateConfigRejectsBadResources(t *testing.T) { + for _, tc := range []struct{ cpu, memory string }{ + {"two", ""}, + {"", "4GB!"}, + {"0", ""}, + {"", "-1Gi"}, + } { + cfg := testTemplateConfig() + cfg.cpu, cfg.memory = tc.cpu, tc.memory + if err := cfg.resolveImages(); err == nil { + t.Errorf("cpu=%q memory=%q: want error, got nil", tc.cpu, tc.memory) + } + } } func TestBuildManifestsCustomAPIPort(t *testing.T) { @@ -291,6 +328,8 @@ func TestManifestTemplateCommandExecution(t *testing.T) { "template", "--guest-image=example.com/guest@sha256:aaaa", "--snapshots-bucket=gs://bucket/prefix/", + "--cpu=4", + "--memory=8Gi", }) if err := cmd.Execute(); err != nil { t.Fatalf("execute manifest template: %v", err) @@ -299,6 +338,11 @@ func TestManifestTemplateCommandExecution(t *testing.T) { if !strings.Contains(out, "image: example.com/guest@sha256:aaaa") || !strings.Contains(out, "storageLocation: gs://bucket/prefix/") { t.Errorf("expected guest image and storage location in output:\n%s", out) } + for _, want := range []string{"sandboxClass: gvisor", "name: cpu\n quantity: \"4\"", "name: memory\n quantity: 8Gi"} { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q:\n%s", want, out) + } + } } const testHex64 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" diff --git a/go.mod b/go.mod index 744476bf..3c937f19 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/agent-substrate/env go 1.27.0 require ( - github.com/agent-substrate/substrate v0.4.1-0.20261013175625-da35a983ed34 + github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d github.com/google/uuid v1.6.0 github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index 11e7bca8..268d930c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/agent-substrate/substrate v0.4.1-0.20261013175625-da35a983ed34 h1:pGiVJjwYbQMDJQj6le/4izVcIz0tHO1i2jQJ8EyKpCU= -github.com/agent-substrate/substrate v0.4.1-0.20261013175625-da35a983ed34/go.mod h1:XVQwhYEmOlb+ILs9BvSCHXYhgo2F6/N2qN4DmRwogQI= +github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d h1:LZW4RXPOy7+7b6K/eNw5jh6udMTdbkG26OuB2VHU1Ag= +github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d/go.mod h1:2Pm1s1pBoB1u4OycVtezqda2BVCjnwcmRQXobWTV3eQ= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index 8da88ee0..25cd9ecc 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -36,10 +36,10 @@ func modeABase() *ateapipb.ActorTemplate { Image: testGuestImage, Command: []string{"/ko-app/ate-env-guest", "-workspace", "/workspace"}, Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, - WakeupProbe: &ateapipb.ContainerWakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, + WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://b/p/", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, Status: &ateapipb.ActorTemplateStatus{}, } } diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index a8791870..13ab41bb 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -540,11 +540,11 @@ func ActorStatusToEnvStatus(st *ateapipb.ActorStatus) ateenvv1alpha.EnvironmentS case ateapipb.ActorState_ACTOR_STATE_SUSPENDING: return ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_SUSPENDING case ateapipb.ActorState_ACTOR_STATE_SUSPENDED: + // Since substrate#2620 a paused actor is SUSPENDED too, with a + // node-local latest snapshot; Substrate has no PAUSED or PAUSING + // state. ate-env never pauses, so ENVIRONMENT_STATUS_PAUSED and + // _PAUSING are no longer produced. return ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_SUSPENDED - case ateapipb.ActorState_ACTOR_STATE_PAUSING: - return ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_PAUSING - case ateapipb.ActorState_ACTOR_STATE_PAUSED: - return ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_PAUSED case ateapipb.ActorState_ACTOR_STATE_CRASHED: return ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_CRASHED case ateapipb.ActorState_ACTOR_STATE_DELETING: diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 87a06c83..81687c06 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -570,8 +570,6 @@ func TestActorStatusToEnvStatus(t *testing.T) { {"running", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_RUNNING}, {"suspending", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_SUSPENDING}, {"suspended", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_SUSPENDED}, - {"pausing", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSING}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_PAUSING}, - {"paused", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_PAUSED}, {"crashed", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_CRASHED}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_CRASHED}, {"deleting", &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_DELETING}, ateenvv1alpha.EnvironmentStatus_ENVIRONMENT_STATUS_DELETING}, } @@ -598,10 +596,10 @@ func seedGuestTemplate(control *fakecontrol.Server, atespace, name string) { Image: testGuestImage, Command: []string{"/ko-app/ate-env-guest"}, Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, - WakeupProbe: &ateapipb.ContainerWakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, + WakeupProbe: &ateapipb.WakeupProbe{HttpGet: &ateapipb.HttpGetAction{Path: "/readyz", Port: 80}}, }}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://bucket/ate-env/"}, - SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, }) } diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go index efda16ec..a699765a 100644 --- a/internal/internaltest/fakecontrol/fakecontrol.go +++ b/internal/internaltest/fakecontrol/fakecontrol.go @@ -299,7 +299,8 @@ func (s *Server) PauseActor(ctx context.Context, req *ateapipb.PauseActorRequest if err != nil { return nil, err } - a.Status = &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_PAUSED} + // Substrate reports a paused actor as SUSPENDED (substrate#2620). + a.Status = &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_SUSPENDED} return &ateapipb.PauseActorResponse{Actor: clone(a)}, nil } From 62ef8399328fd5e139546bdf5f2e34d17d1d81d2 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:42:40 -0700 Subject: [PATCH 2/5] docs: Substrate 6edd1fcc compatibility, ateworker-gvisor, --cpu/--memory The README names the new pin and what the cluster must accept (string sandbox class, CPU and memory limits), says a Substrate older than 2026-10-14 needs an ate-env from before this bump, and documents the --cpu and --memory defaults. The worker image is ateworker-gvisor in the README, docs/release.md and the nemo-gym example. --- README.md | 18 ++++++++++++------ docs/release.md | 2 +- integrations/nemo-gym/README.md | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index beb79be9..0e8cdf61 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ signed, and meant to be used by digest: The digests are in the release notes and in the `images_.txt` asset; the release's `ate-env` binary defaults to them. The worker image -(`--worker-image`, `ateom-gvisor`) comes from the +(`--worker-image`, `ateworker-gvisor`) comes from the [Substrate repo](https://github.com/agent-substrate/substrate) and must match the Substrate version on your cluster. See [docs/release.md](docs/release.md) for verification, pinning and how releases are cut. @@ -65,10 +65,12 @@ verification, pinning and how releases are cut. Prerequisites: a cluster with [Agent Substrate](https://github.com/agent-substrate/substrate) installed and a snapshots bucket. ate-env is built against Substrate -[`da35a983`](https://github.com/agent-substrate/substrate/commit/da35a983) (2026-10-13). The cluster +[`6edd1fcc`](https://github.com/agent-substrate/substrate/commit/6edd1fcc) (2026-10-15). The cluster needs a Substrate recent enough to accept what ate-env writes: the WorkerPool declares -`spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and `wakeupProbe`, and `--worker-image` -must be the `ateom` image published by that Substrate install. +`spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and `wakeupProbe`, names its sandbox +class as the string `gvisor`, and declares CPU and memory limits, and `--worker-image` must be the +`ateworker-gvisor` image published by that Substrate install. A Substrate older than 2026-10-14 rejects +the string sandbox class; use an ate-env from before this bump with it. ### 1. Deploy the Kubernetes Resources @@ -76,12 +78,12 @@ Deploy the namespace, worker pool, and API service: ```bash # With a release binary the API image defaults to the release's. The worker -# image is ateom-gvisor from the Substrate repo at your cluster's version; in +# image is ateworker-gvisor from the Substrate repo at your cluster's version; in # that checkout, `go run ./cmd/ate-setup publish worker-images` pushes it and # prints the digest-pinned ref. Use the digest: a :latest tag is whatever was # built last, and a stale worker image registers workers with no capacity. ate-env manifest \ - --worker-image /ateom-gvisor@sha256: | kubectl apply -f - + --worker-image /ateworker-gvisor@sha256: | kubectl apply -f - # From a source build, add: --api-image ghcr.io/agent-substrate/env/ate-env-api@sha256: # Ensure that the pods are running: @@ -106,6 +108,10 @@ ate-env manifest template \ # From a source build, add: --guest-image ghcr.io/agent-substrate/env/ate-env-guest@sha256: ``` +Each actor gets 2 CPUs and 4 GiB unless you pass `--cpu` and `--memory` (Kubernetes quantities such as +`500m` or `8Gi`). Substrate requires both limits and books them against worker capacity; templates +derived per rootfs image inherit them. + That template's container is the guest image itself. To run another image unmodified, name it with `--rootfs-image`: the guest is then mounted into it as a read-only OCI image volume at `/ate` and started from there, so the rootfs image is never rebuilt. diff --git a/docs/release.md b/docs/release.md index 7532bbcf..2e87d802 100644 --- a/docs/release.md +++ b/docs/release.md @@ -26,7 +26,7 @@ and asks for both flags. `make build-cli` reproduces a release-style binary locally from `ATE_ENV_API_IMAGE` and `ATE_ENV_GUEST_IMAGE`. The worker image (`--worker-image`) is not published here: it is -`ateom-gvisor` from the [Substrate repo](https://github.com/agent-substrate/substrate) +`ateworker-gvisor` from the [Substrate repo](https://github.com/agent-substrate/substrate) and must match the Substrate version deployed on the cluster. From a Substrate checkout at that version, `go run ./cmd/ate-setup publish worker-images` builds and pushes it and prints the digest-pinned ref. Use the digest: a registry's diff --git a/integrations/nemo-gym/README.md b/integrations/nemo-gym/README.md index 214a64cf..5e306f43 100644 --- a/integrations/nemo-gym/README.md +++ b/integrations/nemo-gym/README.md @@ -55,7 +55,7 @@ On the cluster: ate-env manifest \ --guest-image \ --api-image \ - --ateom-image \ + --worker-image \ --snapshots-bucket gs:///ate-env/ | kubectl apply -f - kubectl get pods -n ate-env # api + warm workers Running ``` From 086561e21d75e7178c0a5a8a84e79ad6047c8419 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:49:06 -0700 Subject: [PATCH 3/5] integrations/nemo-gym: split the deploy command into manifest and manifest template The README passed --guest-image and --snapshots-bucket to `ate-env manifest`, which fails with "unknown flag: --guest-image": those flags belong to `ate-env manifest template`. Deploy with `ate-env manifest --api-image --worker-image`, then register the template with `ate-env manifest template --guest-image --snapshots-bucket` once ate-env-api runs, as the root README does. Fixes #76. --- integrations/nemo-gym/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integrations/nemo-gym/README.md b/integrations/nemo-gym/README.md index 5e306f43..68b247dd 100644 --- a/integrations/nemo-gym/README.md +++ b/integrations/nemo-gym/README.md @@ -53,11 +53,13 @@ On the cluster: ```bash ate-env manifest \ - --guest-image \ --api-image \ - --worker-image \ - --snapshots-bucket gs:///ate-env/ | kubectl apply -f - + --worker-image | kubectl apply -f - kubectl get pods -n ate-env # api + warm workers Running + # once ate-env-api is Running (it creates the ate-env atespace templates live in): + ate-env manifest template \ + --guest-image \ + --snapshots-bucket gs:///ate-env/ | kubectl-ate create actor-template -f - ``` If your `ateapi` requires authentication (it does on any standard install), the From 84317c88a6fc5a4557d8cc91d93ab1e1695bf25f Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:40:03 -0700 Subject: [PATCH 4/5] Bump Substrate to 3a2ca623 (2026-10-16): Go module is now go.ate.dev/substrate Substrate main moved its module path to the vanity import go.ate.dev/substrate (substrate#2181, 68a082c3); go.ate.dev serves go-import metadata pointing at the GitHub repository. Every Substrate import moves to the new path, the dependabot rules follow it, and gofmt reorders two import blocks. Nothing else in env changes. Between 6edd1fcc and 3a2ca623 the ateapi proto only gains worker-side fields (Worker.worker_spiffe_id, WorkerAssignment.worker_spiffe_id, MintWorkerActorCertificateRequest.worker); no field env sends or decodes is removed or retyped. WorkerPool keeps its fields; a pool without a ServiceAccount now runs workers as "ateworker", which the controller creates in the pool's namespace (088381b6), so the manifest needs nothing. The renames in 3a2ca623 are in the atelet and ateom protos. The README names the new pin and the exact compatibility boundary: Substrate at or after 63a17a92 (2026-10-15), which accepts the string sandbox class (9ec49759) and requires the limits (c58f374a), and renamed the worker image. --- .github/dependabot.yml | 4 ++-- README.md | 17 +++++++++++------ clients/go/client_test.go | 2 +- cmd/ate-env/manifest.go | 4 ++-- cmd/ate-env/manifest_test.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- internal/apiservice/imagetemplate.go | 2 +- internal/apiservice/imagetemplate_test.go | 2 +- internal/apiservice/server.go | 2 +- internal/apiservice/server_test.go | 2 +- internal/apiservice/wakeretry.go | 2 +- internal/apiservice/wakeretry_test.go | 2 +- internal/ate/client.go | 2 +- internal/ate/client_test.go | 2 +- .../internaltest/fakecontrol/fakecontrol.go | 2 +- internal/mcp/server_test.go | 2 +- 17 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 64038a9d..399ecbfb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,9 +16,9 @@ updates: groups: go-deps: patterns: ["*"] - exclude-patterns: ["github.com/agent-substrate/substrate"] + exclude-patterns: ["go.ate.dev/substrate"] ignore: - - dependency-name: github.com/agent-substrate/substrate + - dependency-name: go.ate.dev/substrate - package-ecosystem: pip directory: /clients/python schedule: diff --git a/README.md b/README.md index 0e8cdf61..4011007e 100644 --- a/README.md +++ b/README.md @@ -65,12 +65,17 @@ verification, pinning and how releases are cut. Prerequisites: a cluster with [Agent Substrate](https://github.com/agent-substrate/substrate) installed and a snapshots bucket. ate-env is built against Substrate -[`6edd1fcc`](https://github.com/agent-substrate/substrate/commit/6edd1fcc) (2026-10-15). The cluster -needs a Substrate recent enough to accept what ate-env writes: the WorkerPool declares -`spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and `wakeupProbe`, names its sandbox -class as the string `gvisor`, and declares CPU and memory limits, and `--worker-image` must be the -`ateworker-gvisor` image published by that Substrate install. A Substrate older than 2026-10-14 rejects -the string sandbox class; use an ate-env from before this bump with it. +[`3a2ca623`](https://github.com/agent-substrate/substrate/commit/3a2ca623) (2026-10-16, Go module +`go.ate.dev/substrate`). The cluster needs a Substrate recent enough to accept what ate-env writes: +the WorkerPool declares `spec.sandboxClasses`, the ActorTemplate uses `snapshotConfig` and +`wakeupProbe`, names its sandbox class as the string `gvisor`, and declares CPU and memory limits. +That means Substrate at or after +[`63a17a92`](https://github.com/agent-substrate/substrate/commit/63a17a92) (2026-10-15), which also +renamed the worker image: `--worker-image` must be the `ateworker-gvisor` image published by that +Substrate install. An older Substrate rejects the string sandbox class (from +[`9ec49759`](https://github.com/agent-substrate/substrate/commit/9ec49759)) or the limits (from +[`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)); use an ate-env from before +this bump with it. ### 1. Deploy the Kubernetes Resources diff --git a/clients/go/client_test.go b/clients/go/client_test.go index 12caa568..56d46170 100644 --- a/clients/go/client_test.go +++ b/clients/go/client_test.go @@ -32,7 +32,7 @@ import ( "github.com/agent-substrate/env/internal/internaltest/fakerouter" "github.com/agent-substrate/env/internal/mcp" ateenvv1alpha "github.com/agent-substrate/env/proto/ateenv/v1alpha" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" ) diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index 7fe96a7e..d3fd519f 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -22,9 +22,9 @@ import ( "strings" "github.com/agent-substrate/env/internal/apiservice" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" appsv1 "k8s.io/api/apps/v1" diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 5a8ca96e..2adf333a 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -20,8 +20,8 @@ import ( "testing" "github.com/agent-substrate/env/internal/apiservice" - atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + atev1alpha1 "go.ate.dev/substrate/pkg/api/v1alpha1" + "go.ate.dev/substrate/pkg/proto/ateapipb" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) diff --git a/go.mod b/go.mod index 3c937f19..f611624d 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/agent-substrate/env go 1.27.0 require ( - github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d github.com/google/uuid v1.6.0 github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/spf13/cobra v1.10.2 + go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748 google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 k8s.io/api v0.37.1 diff --git a/go.sum b/go.sum index 268d930c..edf211fe 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d h1:LZW4RXPOy7+7b6K/eNw5jh6udMTdbkG26OuB2VHU1Ag= -github.com/agent-substrate/substrate v0.5.1-0.20261015061533-6edd1fccfc9d/go.mod h1:2Pm1s1pBoB1u4OycVtezqda2BVCjnwcmRQXobWTV3eQ= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -111,6 +109,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748 h1:QrTauJhnfn0FL0t/GLzVo4bmeVUYMT6e4x8QSaQF8s0= +go.ate.dev/substrate v0.5.1-0.20261016164544-3a2ca6235748/go.mod h1:EktaeScxg724tF4cjtLV8kdz2xN3FH8NOTk/A95jSBY= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= diff --git a/internal/apiservice/imagetemplate.go b/internal/apiservice/imagetemplate.go index bd8f0280..af4d66c2 100644 --- a/internal/apiservice/imagetemplate.go +++ b/internal/apiservice/imagetemplate.go @@ -21,7 +21,7 @@ import ( "regexp" "strings" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index 25cd9ecc..c6988fca 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -18,7 +18,7 @@ import ( "strings" "testing" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/proto" ) diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index 13ab41bb..f3466f60 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -26,7 +26,7 @@ import ( "github.com/agent-substrate/env/internal/ate" ateenvv1alpha "github.com/agent-substrate/env/proto/ateenv/v1alpha" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 81687c06..9ecccd5c 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -27,7 +27,7 @@ import ( "github.com/agent-substrate/env/internal/internaltest/fakecontrol" "github.com/agent-substrate/env/internal/internaltest/fakerouter" ateenvv1alpha "github.com/agent-substrate/env/proto/ateenv/v1alpha" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/insecure" diff --git a/internal/apiservice/wakeretry.go b/internal/apiservice/wakeretry.go index fda16dc9..d9c88571 100644 --- a/internal/apiservice/wakeretry.go +++ b/internal/apiservice/wakeretry.go @@ -21,7 +21,7 @@ import ( "time" ateenvv1alpha "github.com/agent-substrate/env/proto/ateenv/v1alpha" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" diff --git a/internal/apiservice/wakeretry_test.go b/internal/apiservice/wakeretry_test.go index 78ac34b4..19d49265 100644 --- a/internal/apiservice/wakeretry_test.go +++ b/internal/apiservice/wakeretry_test.go @@ -23,7 +23,7 @@ import ( "github.com/agent-substrate/env/guest" "github.com/agent-substrate/env/internal/apiservice" ateenvv1alpha "github.com/agent-substrate/env/proto/ateenv/v1alpha" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" diff --git a/internal/ate/client.go b/internal/ate/client.go index 75af1598..c434e753 100644 --- a/internal/ate/client.go +++ b/internal/ate/client.go @@ -35,7 +35,7 @@ import ( "strings" "sync" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/internal/ate/client_test.go b/internal/ate/client_test.go index bad2250b..727e5fcf 100644 --- a/internal/ate/client_test.go +++ b/internal/ate/client_test.go @@ -23,7 +23,7 @@ import ( "github.com/agent-substrate/env/internal/ate" "github.com/agent-substrate/env/internal/internaltest/fakecontrol" "github.com/agent-substrate/env/internal/internaltest/fakerouter" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) type fixture struct { diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go index a699765a..1a87acc1 100644 --- a/internal/internaltest/fakecontrol/fakecontrol.go +++ b/internal/internaltest/fakecontrol/fakecontrol.go @@ -32,7 +32,7 @@ import ( "sync" "time" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "go.ate.dev/substrate/pkg/proto/ateapipb" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index f069e1ca..3cf48def 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -28,8 +28,8 @@ import ( "github.com/agent-substrate/env/internal/internaltest/fakerouter" internalmcp "github.com/agent-substrate/env/internal/mcp" "github.com/agent-substrate/env/internal/tool" - "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/modelcontextprotocol/go-sdk/mcp" + "go.ate.dev/substrate/pkg/proto/ateapipb" ) func TestMCPServer(t *testing.T) { From 7325ee3a7d4179eb448d6f2eb87a6d86259efbaf Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:47:26 -0700 Subject: [PATCH 5/5] ate-env manifest template: check --cpu/--memory against Substrate's bounds; README upgrade note From review of #91: - --cpu and --memory now follow Substrate's resources.limits validation (internal/resources.ValidateLimit and the Limits.quantity maxLength): a positive quantity of at most 32 characters, cpu at most 999.999 cores, memory at most an int64 byte count. Before, --cpu=1000 printed a template the control plane rejects. The check moves out of resolveImages into validateResources. - An explicit --cpu= or --memory= is an error instead of silently meaning the default. - A test proves that a template derived for a rootfs image keeps the base's limits. - README: after upgrading across the boundary, delete and register base templates again (Substrate has no UpdateActorTemplate, and a base without limits makes each derived template fail resources: Required), and the cpu ceiling next to the --cpu docs. --- README.md | 9 +++- cmd/ate-env/manifest.go | 55 ++++++++++++++++++----- cmd/ate-env/manifest_test.go | 32 ++++++++++++- internal/apiservice/imagetemplate_test.go | 18 +++++++- 4 files changed, 101 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4011007e..d80333a6 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,13 @@ Substrate install. An older Substrate rejects the string sandbox class (from [`c58f374a`](https://github.com/agent-substrate/substrate/commit/c58f374a)); use an ate-env from before this bump with it. +When you upgrade an existing cluster across that boundary, register your base templates again. +Substrate cannot update an ActorTemplate in place, and a base registered earlier has no CPU and memory +limits, thus each template `ate-env-api` derives from it for a rootfs image fails with +`resources: Required`. Delete the base and its derived templates (named `-<12 hex digits>`) with +`kubectl-ate delete actor-template`, then register the base again with `ate-env manifest template`. +Each environment created after that derives its template again, with the limits. + ### 1. Deploy the Kubernetes Resources Deploy the namespace, worker pool, and API service: @@ -114,7 +121,7 @@ ate-env manifest template \ ``` Each actor gets 2 CPUs and 4 GiB unless you pass `--cpu` and `--memory` (Kubernetes quantities such as -`500m` or `8Gi`). Substrate requires both limits and books them against worker capacity; templates +`500m` or `8Gi`; CPU at most `999.999`, as Substrate requires). Substrate requires both limits and books them against worker capacity; templates derived per rootfs image inherit them. That template's container is the guest image itself. To run another image unmodified, name diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index d3fd519f..9552aec2 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -153,19 +153,30 @@ func parseLayers(values []string) ([]runtimeLayer, error) { return layers, nil } -func (c *templateConfig) resolveImages() error { - if c.guestImage == "" { - return errors.New(`--guest-image is required when ate-env is built from source; release -binaries default it to the ate-env-guest image published with the release -(see docs/release.md), or build and push your own with make images`) - } - if c.snapshotsBucket == "" { - return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") - } - for _, l := range []struct{ flag, value string }{{"--cpu", c.cpu}, {"--memory", c.memory}} { +// Bounds Substrate applies to an ActorTemplate's limits +// (go.ate.dev/substrate/internal/resources.ValidateLimit and the +// Limits.quantity maxLength), checked here so the CLI never prints a template +// the control plane rejects. +const maxQuantityLen = 32 + +var ( + maxCPULimit = resource.MustParse("999.999") // the wire carries millicores below 1000 cores + maxMemoryLimit = resource.MustParse("9223372036854775807") // the wire carries an int64 byte count +) + +// validateResources checks --cpu and --memory the way Substrate validates +// resources.limits. An empty value means the default. +func (c *templateConfig) validateResources() error { + for _, l := range []struct { + flag, value string + max resource.Quantity + }{{"--cpu", c.cpu, maxCPULimit}, {"--memory", c.memory, maxMemoryLimit}} { if l.value == "" { continue } + if len(l.value) > maxQuantityLen { + return fmt.Errorf("%s %q is longer than %d characters, which Substrate rejects", l.flag, l.value, maxQuantityLen) + } q, err := resource.ParseQuantity(l.value) if err != nil { return fmt.Errorf("%s %q is not a Kubernetes quantity (e.g. 2, 500m, 4Gi): %w", l.flag, l.value, err) @@ -173,6 +184,21 @@ binaries default it to the ate-env-guest image published with the release if q.Sign() <= 0 { return fmt.Errorf("%s must be greater than zero, got %q", l.flag, l.value) } + if q.Cmp(l.max) > 0 { + return fmt.Errorf("%s %q is above Substrate's maximum of %s", l.flag, l.value, l.max.String()) + } + } + return nil +} + +func (c *templateConfig) resolveImages() error { + if c.guestImage == "" { + return errors.New(`--guest-image is required when ate-env is built from source; release +binaries default it to the ate-env-guest image published with the release +(see docs/release.md), or build and push your own with make images`) + } + if c.snapshotsBucket == "" { + return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") } if c.rootfsImage != "" { if _, err := apiservice.ImageDigest(c.rootfsImage); err != nil { @@ -258,6 +284,15 @@ It prints YAML to stdout without touching the cluster.`, if err := tCfg.resolveImages(); err != nil { return err } + // An empty --cpu or --memory would silently mean the default. + for _, f := range []string{"cpu", "memory"} { + if cmd.Flags().Changed(f) && cmd.Flags().Lookup(f).Value.String() == "" { + return fmt.Errorf("--%s must not be empty; omit it for the default", f) + } + } + if err := tCfg.validateResources(); err != nil { + return err + } tmpl, err := buildTemplate(tCfg) if err != nil { return err diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 2adf333a..95f26594 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -206,13 +206,43 @@ func TestTemplateConfigRejectsBadResources(t *testing.T) { {"", "4GB!"}, {"0", ""}, {"", "-1Gi"}, + // Substrate's bounds: cpu below 1000 cores, memory an int64 byte + // count, each quantity at most 32 characters. + {"1000", ""}, + {"1e3", ""}, + {"999.9991", ""}, + {"", "9223372036854775808"}, + {"0.0000000000000000000000000000001", ""}, } { cfg := testTemplateConfig() cfg.cpu, cfg.memory = tc.cpu, tc.memory - if err := cfg.resolveImages(); err == nil { + if err := cfg.validateResources(); err == nil { t.Errorf("cpu=%q memory=%q: want error, got nil", tc.cpu, tc.memory) } } + for _, tc := range []struct{ cpu, memory string }{ + {"999.999", "9223372036854775807"}, + {"500m", "8Gi"}, + {"", ""}, + } { + cfg := testTemplateConfig() + cfg.cpu, cfg.memory = tc.cpu, tc.memory + if err := cfg.validateResources(); err != nil { + t.Errorf("cpu=%q memory=%q: %v, want accepted", tc.cpu, tc.memory, err) + } + } +} + +func TestManifestTemplateCommandRejectsEmptyResource(t *testing.T) { + for _, flag := range []string{"--cpu=", "--memory="} { + cmd := newManifestCommand() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{"template", "--guest-image=example.com/guest@sha256:aaaa", "--snapshots-bucket=gs://b/", flag}) + if err := cmd.Execute(); err == nil || !strings.Contains(err.Error(), "must not be empty") { + t.Errorf("%s: got %v, want an error about the empty value", flag, err) + } + } } func TestBuildManifestsCustomAPIPort(t *testing.T) { diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index c6988fca..d7050e23 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -40,7 +40,23 @@ func modeABase() *ateapipb.ActorTemplate { }}, SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://b/p/", PreferredFidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY}, SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: "gvisor", ConfigName: "gvisor-default"}, - Status: &ateapipb.ActorTemplateStatus{}, + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{ + {Name: "cpu", Quantity: "2"}, {Name: "memory", Quantity: "4Gi"}, + }}, + Status: &ateapipb.ActorTemplateStatus{}, + } +} + +// Substrate requires cpu and memory limits on every ActorTemplate, thus a +// derived template must carry the base's limits unchanged. +func TestDeriveImageTemplateKeepsResources(t *testing.T) { + base := modeABase() + got, err := DeriveImageTemplate(base, "default-template-0123456789ab", testRootfsImage) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(got.GetResources(), base.GetResources()) { + t.Errorf("derived resources = %v, want the base's %v", got.GetResources(), base.GetResources()) } }