diff --git a/benchmarking/README.md b/benchmarking/README.md index b8bc2aef47..f4e7540ccc 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -371,9 +371,10 @@ deployment fails loudly instead of showing up as OOM-flaky steps. gap gets ±20% jitter so sessions don't move in lockstep. * `--resume-mode implicit|explicit` — implicit (default) lets the parked first request wake the actor; explicit issues ResumeActor before traffic. -* `--lifecycle-mode suspend|pause|none` — durable suspend (default), - node-local pause between steps, or `none` to keep actors resident in memory - between steps without hibernating. +* `--lifecycle-mode suspend|pause|none` — synchronous suspend (`suspend`, + default, uploads the snapshot before returning), asynchronous suspend + (`pause`, uploads the snapshot in the background), or `none` to keep actors + resident in memory between steps without hibernating. * `--agentsession-snapshot-steps` — comma-separated 1-based step numbers after which the actor is hibernated (for example, `3,10,20,30,40,50,61`); between all other steps the actor stays resident. @@ -539,26 +540,26 @@ count users × `--actors-per-user`, not `--total-actors`. ### ResumePaused Benchmark The ResumePaused benchmark drives ate-api-server's cold resume path at a set -rate. A cold resume here is a ResumeActor of a paused actor: ateapi takes the +rate. A cold resume here is a ResumeActor of a suspended actor: ateapi takes the actor lease, binds a worker, and restores the actor from its snapshot. A ResumeActor of a running actor does none of that. Each user owns one actor from the `glutton` template. On its first iteration the user creates the actor -and makes it cold with a resume and a pause; every later iteration resumes the -actor and pauses it again, so every measured resume is cold. +and makes it cold with a resume and a suspend; every later iteration resumes the +actor and suspends it again, so every measured resume is cold. The warm-up runs in the step that added the user, unpaced, and is recorded under its own names. It sends no requests to the actors themselves, so it runs on the fake data plane, where it measures what a cold resume costs ate-api-server and Postgres -without the router or the restore in the way. The pauses run at the resume -rate, so a capacity it finds is one of resume and pause cycles, not of resumes +without the router or the restore in the way. The suspends run at the resume +rate, so a capacity it finds is one of resume and suspend cycles, not of resumes alone. Actors are named `rp-` and deleted when boomer exits. If the atespace cannot be ensured, boomer exits at startup. An actor that fails to warm is deleted, and its user creates another on its next iteration. When a step lowers -the user count, the actors of the stopped users stay paused, unused, until +the user count, the actors of the stopped users stay suspended, unused, until boomer exits. #### ResumePaused Configuration Knobs @@ -566,7 +567,7 @@ boomer exits. * `--target-rps`: Cold resumes per second the whole boomer process paces to (default `0`, unpaced). A `--ladder` step ending in `@target_rps`, as in `200:3m@1000`, changes it as the step starts. Keep the users at or above the - rate times the p99 of one resume plus one pause in seconds, or the step falls + rate times the p99 of one resume plus one suspend in seconds, or the step falls short of its rate. The web UI shows the same field. @@ -792,9 +793,9 @@ actually did, independent of what the load generator reported. * `snapshots.restore_p50_s` through `restore_p99_s`, `restore_mean_s`, and the same for `checkpoint_*`: how long the atelet took to resume or suspend an actor (`ate_actor_{restore,checkpoint}_duration`, `total` phase). Every - snapshot kind counts, as in the sizes: the actor's own snapshot, a - template's first start from its golden snapshot, and a pause. The buckets - reach 60s, so a slower operation reads as 60s. + snapshot kind counts, as in the sizes: the actor's own snapshot and a + template's first start from its golden snapshot. The buckets reach 60s, so a + slower operation reads as 60s. * `snapshots.checkpoints_in_window`, `checkpoints_cumulative`: checkpoint volume over the steady-state window, and since the atelet started. * `snapshots.checkpoint_mb_s`: bytes written per second spent checkpointing, diff --git a/benchmarking/locust/common/lifecycle_mode.py b/benchmarking/locust/common/lifecycle_mode.py index 644a50df40..77e27de69a 100644 --- a/benchmarking/locust/common/lifecycle_mode.py +++ b/benchmarking/locust/common/lifecycle_mode.py @@ -26,7 +26,7 @@ def add_lifecycle_mode_arguments(parser: LocustArgumentParser) -> None: type=str, default="suspend", choices=["suspend", "pause", "none"], - help="Lifecycle mode: 'suspend' uses SuspendActor (persists snapshot to storage); " - "'pause' uses an asynchronous SuspendActor (keeps snapshot on local node); " + help="Lifecycle mode: 'suspend' uses synchronous SuspendActor (uploads snapshot before returning); " + "'pause' uses asynchronous SuspendActor (uploads snapshot in background); " "'none' keeps actors resident in memory between steps (default: suspend)", ) diff --git a/cmd/ateapi/internal/controlapi/actor_test.go b/cmd/ateapi/internal/controlapi/actor_test.go index d10967a0cf..745abac436 100644 --- a/cmd/ateapi/internal/controlapi/actor_test.go +++ b/cmd/ateapi/internal/controlapi/actor_test.go @@ -361,8 +361,8 @@ func TestUpdateActor_RepointTemplate(t *testing.T) { t.Fatalf("UpdateActor repointing a running actor = %v, want FailedPrecondition (err: %v)", got, err) } - // Repointing a paused actor is allowed while suspended, even when it holds - // a node-local snapshot. + // Repointing a suspended actor is allowed even when it holds a node-local + // snapshot. for _, name := range []string{"paused-actor", "uploaded-paused-actor"} { paused := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: name}, @@ -468,10 +468,10 @@ func TestUpdateActor_RepointTemplateStorageLocation(t *testing.T) { name string template string snapshot func(*testing.T, *ateapipb.Actor) string - // pending leaves the snapshot's upload PENDING, as a pause does. + // pending leaves the snapshot's upload PENDING, as an async suspend does. pending bool // lastAssignedGen overrides status.LastAssignedGeneration when > 0 - // (e.g. after a reverted pause or crashed suspend pruned a higher + // (e.g. after a reverted or crashed suspend pruned a higher // pending generation). lastAssignedGen int32 repointTo string diff --git a/cmd/ateapi/internal/controlapi/export_test.go b/cmd/ateapi/internal/controlapi/export_test.go index 3d0a435cac..daeed181c6 100644 --- a/cmd/ateapi/internal/controlapi/export_test.go +++ b/cmd/ateapi/internal/controlapi/export_test.go @@ -53,7 +53,7 @@ type ReleaseWindowWorkflow struct { // Snapshot, when set, is the actor's latest snapshot. Snapshot func(t *testing.T) *ateapipb.Snapshot // Finalize runs the workflow or step that releases the worker and then - // records the actor's new state against st. Pause and suspend reach the + // records the actor's new state against st. Async and sync suspend reach the // fake atelet newWireCaptureWorkflow serves for the worker // WireTestAssignment names, so the actor must be assigned that worker // and run from the template SeedSuspendTemplate stores. @@ -64,11 +64,11 @@ type ReleaseWindowWorkflow struct { WantFinalState ateapipb.ActorState } -// ReleaseWindowWorkflows returns pause, suspend, crash, and a resume onto an -// ineligible worker: the workflows that release a worker before rewriting the -// actor's status while the actor is in a state HostingNode resolves through -// the worker assignment. Revert and delete release theirs in REVERTING and -// DELETING, which HostingNode never resolves to a node. +// ReleaseWindowWorkflows returns async suspend ("pause"), sync suspend, crash, +// and a resume onto an ineligible worker: the workflows that release a worker +// before rewriting the actor's status while the actor is in a state HostingNode +// resolves through the worker assignment. Revert and delete release theirs in +// REVERTING and DELETING, which HostingNode never resolves to a node. func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { // The workers these tests create carry no labels, so a template // selecting a pool makes any of them ineligible. @@ -80,10 +80,10 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Name: "pause", State: ateapipb.ActorState_ACTOR_STATE_SUSPENDING, Snapshot: func(t *testing.T) *ateapipb.Snapshot { - // A pause in progress: a PENDING durable copy, not yet captured. + // An async suspend in progress: a PENDING durable copy, not yet captured. return newInProgressSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "local-snap-1") }, - // The actor is seeded SUSPENDING, so the pause skips its mark step + // The actor is seeded SUSPENDING, so the suspend skips its mark step // and its first actor update is the finalize commit. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) @@ -99,7 +99,7 @@ func ReleaseWindowWorkflows() []ReleaseWindowWorkflow { Snapshot: func(t *testing.T) *ateapipb.Snapshot { return newDurableSnapshot(1, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "snapshot-1", someActorSnapshotURI(t, testStorageLocation, "team-a", "snapshot-1"), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) }, - // As for the pause, the finalize commit is the first actor update. + // As for the async suspend, the finalize commit is the first actor update. Finalize: func(t *testing.T, ctx context.Context, st store.Interface, actorRef resources.ActorRef, _ *ateapipb.Actor) error { w, _ := newWireCaptureWorkflow(t, st) _, err := w.SuspendActor(ctx, actorRef, suspendSync) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index 067fdda317..cf88107845 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -1512,9 +1512,9 @@ func TestDeleteActor_StateDeleting(t *testing.T) { } } -// TestDeleteActor_Paused deletes a paused actor: a paused actor is SUSPENDED -// (holding a node-local snapshot), and the delete terminates it on its -// assigned node so the local snapshot is removed. +// TestDeleteActor_Paused deletes an asynchronously suspended actor: it is +// SUSPENDED (holding a node-local snapshot), and the delete terminates it on +// its assigned node so the local snapshot is removed. func TestDeleteActor_Paused(t *testing.T) { ns := namespaceForTest("ns-delete-paused") tc := setupTest(t, ns) @@ -1862,7 +1862,7 @@ func TestActorLifecycle_WithExternalVolumes(t *testing.T) { } waitForWorkerAvailable(t, tc, workerName) - // 4. ResumeActor from paused + // 4. ResumeActor from node-local snapshot waitForWorkerAvailable(t, tc, workerName) resumeResp2, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-vol-lc"}, @@ -2861,7 +2861,7 @@ func TestSuspendActor_VolumeDetachFailure_DeleteActor(t *testing.T) { // 1. Creates a template with an external volume and a mock worker pod on node1. // 2. Creates and resumes an actor to ACTOR_STATE_RUNNING; verifies volume is attached to node1 and restore request carries node1 publishContext. // 3. Calls async SuspendActor; verifies volume is detached from node1 and actor transitions to ACTOR_STATE_SUSPENDED with a local snapshot. -// 4. Calls ResumeActor from pause; verifies volume is re-attached to node1 and restore request carries node1 publishContext. +// 4. Calls ResumeActor from local snapshot; verifies volume is re-attached to node1 and restore request carries node1 publishContext. // 5. Calls SuspendActor, replaces worker-1 (node1) with worker-2 (node2), and calls ResumeActor; verifies volume is attached to node2 and restore request carries node2 publishContext. // 6. Calls SuspendActor and DeleteActor; verifies volume is detached from node2 and deleted. func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) { @@ -2945,7 +2945,7 @@ func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) plugin.mu.Unlock() waitForWorkerAvailable(t, tc, workerName) - // 4. ResumeActor from pause: volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. + // 4. ResumeActor from local snapshot: volume is re-attached to node1 and actor transitions to ACTOR_STATE_RUNNING. tc.fakeAtelet.Reset() _, err = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "pause-vol-actor"}, @@ -2964,7 +2964,7 @@ func TestSuspendActorAsync_VolumeLifecycle_DetachResumeAndMigrate(t *testing.T) t.Errorf("actor state after retry = %v, want ACTOR_STATE_RUNNING", getResp.GetStatus().GetState()) } plugin.mu.Lock() - // Should have attached twice: first resume + resume from pause + // Should have attached twice: first resume + resume from local snapshot if diff := cmp.Diff([]string{"node1", "node1"}, plugin.attachedNodes); diff != "" { t.Errorf("attached nodes after resume from pause mismatch (-want +got):\n%s", diff) } @@ -3780,7 +3780,7 @@ func TestResumeActor_AteletTransportErrorLeavesActorResuming(t *testing.T) { } // TestResumeActor_LocalRestoreFailureCrashesActor: an atelet error restoring a -// paused actor from its local snapshot crashes it and releases its worker. +// suspended actor from its local snapshot crashes it and releases its worker. func TestResumeActor_LocalRestoreFailureCrashesActor(t *testing.T) { ns := namespaceForTest("ns-resume-local-crash") tc := setupTest(t, ns) @@ -4425,9 +4425,9 @@ func TestResumeActor_RepointTemplateBeforeResume(t *testing.T) { } // TestResumeActor_PausedAfterRepointUsesLocalProvenance verifies that an actor -// paused after a template repoint restores from its pause checkpoint in FULL, -// not DATA. At that point the actor holds an external snapshot captured on v1 -// and a local checkpoint captured on v2; judging the local restore by the +// suspended after a template repoint restores from its local checkpoint in +// FULL, not DATA. At that point the actor holds an external snapshot captured +// on v1 and a local checkpoint captured on v2; judging the local restore by the // external snapshot's provenance would wrongly discard the v2 memory image. func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { ns := namespaceForTest("ns-repoint-pause") @@ -4469,8 +4469,8 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } // Resume on v2 (this restore is DATA, from the v1 external snapshot), then - // pause; the pause checkpoint is captured on v2 while the external - // snapshot still says v1. + // suspend asynchronously; the local checkpoint is captured on v2 while the + // external snapshot still says v1. if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(v2 from v1 snapshot) failed: %v", err) } @@ -4487,8 +4487,8 @@ func TestResumeActor_PausedAfterRepointUsesLocalProvenance(t *testing.T) { } waitForWorkerAvailable(t, tc, worker) - // Resume from pause: the local checkpoint was captured on v2 and the - // actor's template is v2, so the restore must be FULL. + // Resume from local snapshot: the local checkpoint was captured on v2 and + // the actor's template is v2, so the restore must be FULL. if _, err := tc.client.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("ResumeActor(from v2 pause) failed: %v", err) } @@ -4971,9 +4971,9 @@ func TestSuspendActorAsynchronous(t *testing.T) { } // TestResumeActor_PausedLocalSnapshotMissing_Crashes tests that if the local checkpoint -// files on the worker node are missing (e.g. node reboot, /tmp wipe) when resuming a paused actor, -// atelet returns a terminal file system error and ateapi marks the actor ACTOR_STATE_CRASHED -// while releasing the assigned worker pod. +// files on the worker node are missing (e.g. node reboot, /tmp wipe) when resuming a +// suspended actor from its local snapshot, atelet returns a terminal file system error +// and ateapi marks the actor ACTOR_STATE_CRASHED while releasing the assigned worker pod. func TestResumeActor_PausedLocalSnapshotMissing_Crashes(t *testing.T) { ns := namespaceForTest("ns-resume-paused-missing") tc := setupTest(t, ns) @@ -5588,8 +5588,8 @@ func TestSuspendActor_DanglingWorker(t *testing.T) { } // TestSuspendActor_FromPaused verifies SuspendActor, in either mode, returns a -// paused actor as it is: atelet is not called, and the actor keeps its -// node-local snapshot and stays pinned to its node. +// suspended actor as it is: atelet is not called, and the actor keeps its +// node-local snapshot on its node. func TestSuspendActor_FromPaused(t *testing.T) { tests := []struct { name string @@ -5630,8 +5630,8 @@ func TestSuspendActor_FromPaused(t *testing.T) { if localSnapshot(paused.GetStatus()) == nil { t.Fatal("paused actor has no local snapshot") } - // Drop the pause's Checkpoint call so the suspend's atelet traffic is - // observable in isolation. + // Drop the first suspend's Checkpoint call so the second suspend's + // atelet traffic is observable in isolation. tc.fakeAtelet.Reset() suspended, err := tc.client.SuspendActor(context.Background(), &ateapipb.SuspendActorRequest{Actor: ref, Options: tt.opts}) @@ -5847,8 +5847,8 @@ func TestSuspendActor_SnapshotConflictPreservesState(t *testing.T) { } } -// TestCheckpointFailureCrashes: an atelet Checkpoint error while pausing or -// suspending a running actor crashes it, records atelet's error text, and +// TestCheckpointFailureCrashes: an atelet Checkpoint error while suspending a +// running actor (in either mode) crashes it, records atelet's error text, and // releases its worker. func TestCheckpointFailureCrashes(t *testing.T) { tests := []struct { @@ -5928,9 +5928,9 @@ func TestCheckpointFailureCrashes(t *testing.T) { } // TestResumeActor_RelocatesAfterDetach covers the capacity-recovery flow: a -// paused actor is pinned to the node holding its local snapshot, so it cannot -// resume while that node is full. Once atelet reports the snapshot uploaded and -// detached from the node, the pin is dropped and the actor is scheduled onto a +// suspended actor with a local snapshot is placed on the node holding its +// local snapshot. Once atelet reports the snapshot uploaded and detached from +// the node, the node placement is cleared and the actor is scheduled onto a // worker on a different node. func TestResumeActor_RelocatesAfterDetach(t *testing.T) { ns := namespaceForTest("ns-resume-relocate") @@ -5950,8 +5950,9 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { } } - // The actor under test runs on node1's only worker, then pauses — which - // frees that worker but pins the actor to node1 via LocalSnapshot. + // The actor under test runs on node1's only worker, then suspends + // asynchronously — which frees that worker but pins the actor to node1 via + // the local snapshot. if _, err := tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: pinned}, }); err != nil { @@ -5989,7 +5990,7 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { createWorkerPod(t, tc, ns, "worker-2", "node2", "pool1") setupAteletOnNode(t, tc, "atelet-node2", "node2") - // node1's atelet reports the paused snapshot uploaded and detached: a report + // node1's atelet reports the local snapshot uploaded and detached: a report // with only latest_durable. ls := localSnapshot(paused.GetStatus()) if err := tc.service.ApplyActorSnapshotState(context.Background(), resources.ActorRefFromActor(paused), paused.GetMetadata().GetUid(), "node1", &ateapipb.ActorSnapshotState{ @@ -6034,7 +6035,7 @@ func TestResumeActor_RelocatesAfterDetach(t *testing.T) { } } -// TestResumeActor_DetachesFromFullNode resumes a paused actor whose node is +// TestResumeActor_DetachesFromFullNode resumes a suspended actor whose node is // full while another has room: it is detached and restored on the other node. func TestResumeActor_DetachesFromFullNode(t *testing.T) { ns := namespaceForTest("ns-resume-detach") @@ -6108,7 +6109,7 @@ func TestResumeActor_DetachesFromFullNode(t *testing.T) { } // TestLifecycleOpPoolAttributesOnSuccess is the regression test for #957: a -// successful suspend and pause must stamp the pool they ran on. Both recorded +// successful sync or async suspend must stamp the pool it ran on. Both recorded // the histogram from a defer that read the finalized record, whose assignment // the finalize step had already cleared, so the pair landed only on failures. func TestLifecycleOpPoolAttributesOnSuccess(t *testing.T) { @@ -6544,8 +6545,9 @@ func TestRevertActor_UnreachableNode(t *testing.T) { } } -// TestRevertActor_FromPaused reverts a paused actor back to its external -// snapshot, discarding the local pause snapshot info on the actor record. +// TestRevertActor_FromPaused reverts a suspended actor with a local snapshot +// back to its external snapshot, discarding the local snapshot info on the +// actor record. func TestRevertActor_FromPaused(t *testing.T) { ns := namespaceForTest("ns-revert-paused") tc := setupTest(t, ns) @@ -7001,7 +7003,8 @@ func reportWorkerRuntime(t *testing.T, tc *testContext, workerName string, defau // 4. When the other worker advertises the snapshot's runtime in // restorable_runtimes, ResumeActor restores at full MEMORY fidelity, // records the matched restorable runtime on WorkerAssignment, and preserves -// that stamp through async SuspendActor and sync SuspendActor-from-PAUSED. +// that stamp through async SuspendActor and sync SuspendActor on an +// already-SUSPENDED actor. func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { ns := namespaceForTest("ns-runtime-fallback") tc := setupTest(t, ns) @@ -7222,7 +7225,8 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { // keeping rtB as its default. Resuming an actor whose snapshot requires rtA // onto worker-b (with worker-a still full) must restore at MEMORY fidelity, // record the matched restorable entry (rtA, not rtB) on WorkerAssignment, - // and preserve rtA through async SuspendActor and sync SuspendActor-from-PAUSED. + // and preserve rtA through async SuspendActor and sync SuspendActor on the + // already-SUSPENDED actor. if _, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenActorRef}); err != nil { t.Fatalf("SuspendActor(golden-actor) failed: %v", err) } @@ -7260,8 +7264,8 @@ func TestResumeActor_SandboxRuntimePlacementAndFallback(t *testing.T) { t.Fatalf("paused local Snapshot.SandboxRuntime = %v, want %v", got, rtA) } - // A sync suspend of the paused actor returns it as it is: the node-local - // snapshot keeps its runtime stamp. + // A sync suspend of the already-SUSPENDED actor returns it as it is: the + // node-local snapshot keeps its runtime stamp. suspendedFromPaused, err := tc.client.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: restorableRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_SYNCHRONOUS}}) if err != nil { t.Fatalf("SuspendActor from PAUSED(restorable-actor) failed: %v", err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 1c00dcc2a3..2e7143d3ef 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -533,8 +533,9 @@ func newDurableSnapshot(gen int32, owner ateapipb.SnapshotOwner, fidelity ateapi } } -// newPausedSnapshot is the snapshot a pause leaves on node locality: its -// durable copy, under the actor's own prefix, is PENDING until atelet uploads it. +// newPausedSnapshot is the snapshot an asynchronous suspend leaves on node +// locality: its durable copy, under the actor's own prefix, is PENDING until +// atelet uploads it. func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, actorUID, uuid, locality string) *ateapipb.Snapshot { t.Helper() uri, err := resources.NewActorSnapshotURI(testStorageLocation, testAtespace, actorUID, uuid) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go index 9249c50888..b72de3f26b 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/tag_test.go @@ -151,9 +151,10 @@ func TestCreateTag_ReusedTagName(t *testing.T) { } } -// TestCreateTag_FromPausedActor verifies that tagging a paused actor has the -// atelet on its node upload the node-local snapshot first, and tags that -// snapshot. The actor keeps the snapshot on its node. +// TestCreateTag_FromPausedActor verifies that tagging a suspended actor whose +// local snapshot is not uploaded yet has the atelet on its node upload the +// node-local snapshot first, and tags that snapshot. The actor keeps the +// snapshot on its node. func TestCreateTag_FromPausedActor(t *testing.T) { ns := namespaceForTest("ns-tag-paused") tc := setupTest(t, ns) @@ -209,7 +210,7 @@ func TestCreateTag_FromPausedActor(t *testing.T) { if err != nil { t.Fatalf("GetActor failed: %v", err) } - // The paused snapshot is now durable and stays on its node. + // The node-local snapshot is now durable and stays on its node. if got := durableSnapshot(stored.GetStatus()); got.GetUuid() != pausedSnap.GetUuid() { t.Errorf("latest durable snapshot = %v, want the paused snapshot %q", got, pausedSnap.GetUuid()) } diff --git a/cmd/ateapi/internal/controlapi/hosting.go b/cmd/ateapi/internal/controlapi/hosting.go index b155c9a7f7..d93afee96f 100644 --- a/cmd/ateapi/internal/controlapi/hosting.go +++ b/cmd/ateapi/internal/controlapi/hosting.go @@ -27,7 +27,7 @@ import ( // HostingNode is the node that holds the actor, and "" when no node does. The // actor's state decides which status field to trust, since both can outlive -// the hosting they record: pause, suspend, crash and revert release the +// the hosting they record: suspend, crash and revert release the // worker (delete its row in the store's worker assignments) before they clear // status.worker_assignment, and a crash keeps status.assigned_node as a record // of the last node. The rules: diff --git a/cmd/ateapi/internal/controlapi/hosting_test.go b/cmd/ateapi/internal/controlapi/hosting_test.go index cdbff1e107..a963441391 100644 --- a/cmd/ateapi/internal/controlapi/hosting_test.go +++ b/cmd/ateapi/internal/controlapi/hosting_test.go @@ -36,13 +36,13 @@ import ( // doing so until the workflow is retried. The worker's node must not count as // hosting the actor in that window, nor once the worker hosts another actor. func TestHostingNodeAfterWorkerRelease(t *testing.T) { - // The worker is the one wireTestAssignment names, so a pause or suspend + // The worker is the one wireTestAssignment names, so an async or sync suspend // reaches newWireCaptureWorkflow's fake atelet on its node. wa := wireTestAssignment() node := wa.GetNodeName() // wantFinalNode is HostingNode once the retry commits, by workflow name. wantFinalNode := map[string]string{ - // Paused: the node holding the local snapshot. + // Async suspend: the node holding the local snapshot. "pause": node, } for _, tc := range ReleaseWindowWorkflows() { diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index f9e4e44438..647045720d 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -202,13 +202,13 @@ func (i *Instruments) recordLifecycleOp(ctx context.Context, op string, start ti i.lifecycleOpDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...)) } -// lifecycleOpAttrs builds the resume/suspend/pause/revert dimensions from +// lifecycleOpAttrs builds the resume/suspend/revert dimensions from // workflow state. Nil-safe, and omits the template, pool, sandbox-class, // snapshot-kind and snapshot-fidelity labels until they are known so a failure // before the load/assign/restore steps never emits an empty-string series. -// snapshotKind is empty for suspend/pause, which do not restore; snapshotScope -// applies to all three and separates a full restore from a data-only one of the -// same kind. +// snapshotKind is empty for suspend, which does not restore; snapshotScope +// applies to resume and suspend and separates a full restore from a data-only +// one of the same kind. // The pool keys are set together or not at all; see ateattr.WorkerPoolAttributes. func lifecycleOpAttrs(actor *ateapipb.Actor, snapshotKind, snapshotScope string) []attribute.KeyValue { var attrs []attribute.KeyValue diff --git a/cmd/ateapi/internal/controlapi/mint_after_release_test.go b/cmd/ateapi/internal/controlapi/mint_after_release_test.go index f638c8f63b..a33c8a59ba 100644 --- a/cmd/ateapi/internal/controlapi/mint_after_release_test.go +++ b/cmd/ateapi/internal/controlapi/mint_after_release_test.go @@ -45,7 +45,7 @@ import ( // window, nor once it hosts another actor or the status is finalized, and // another worker on the node never gets one. func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { - // The worker is the one WireTestAssignment names, so a pause or suspend + // The worker is the one WireTestAssignment names, so an async or sync suspend // reaches the workflow's fake atelet on its node. wa := controlapi.WireTestAssignment() node := wa.GetNodeName() @@ -168,7 +168,7 @@ func TestMintWorkerActorCertificateAfterWorkerRelease(t *testing.T) { if actor.GetStatus().GetState() != wf.WantFinalState { t.Errorf("actor state once finalized = %v, want %v", actor.GetStatus().GetState(), wf.WantFinalState) } - // No worker runs the actor once finalized, paused or not. + // No worker runs the actor once finalized, with or without a local snapshot. if got != codes.NotFound { t.Errorf("certificate mint once finalized: code %v, want NotFound", got) } diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers.go b/cmd/ateapi/internal/controlapi/snapshot_helpers.go index 03b63e43fd..45c027d7ed 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers.go @@ -91,13 +91,14 @@ func clearLocalSnapshots(status *ateapipb.ActorStatus) { // beforehand. // // Dropping PENDING durable snapshots is safe because: -// - On pause finalization, the new snapshot is kept with its PENDING copy. -// An earlier pause's PENDING copy is dropped: whatever atelet uploaded of -// it stays under the actor's prefix until DeleteActor collects it. +// - On asynchronous suspend finalization, the new snapshot is kept with its +// PENDING copy. An earlier asynchronous suspend's PENDING copy is dropped: +// whatever atelet uploaded of it stays under the actor's prefix until +// DeleteActor collects it. // - On revert finalization, ensureInProgressSnapshotDiscarded has already // deleted the objects of every PENDING durable snapshot. -// - On suspend finalization, the latest snapshot has already been marked -// COMPLETED. +// - On synchronous suspend finalization, the latest snapshot has already been +// marked COMPLETED. // // status.LastAssignedGeneration is not decreased so that we can keep track of // stale snapshots that need to be cleaned up and discarded generation numbers diff --git a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go index 207559c4d0..2f466f6329 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_helpers_test.go @@ -74,7 +74,7 @@ func TestFindLatestDurableAndLocalSnapshots(t *testing.T) { t.Errorf("findLatestDurableSnapshot(COMPLETED) = %v, want gen 2 (gs://b/snap-2)", gotSnap) } - // A paused snapshot's durable copy is PENDING until atelet uploads it. + // An async suspend snapshot's durable copy is PENDING until atelet uploads it. gotSnap = findLatestDurableSnapshot(status, ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) if gotSnap != s4 { t.Errorf("findLatestDurableSnapshot(PENDING) = %v, want gen 4", gotSnap) @@ -207,10 +207,9 @@ func TestSnapshotPruningHelpers(t *testing.T) { }) } -// TestLatestSnapshotIsLocal pins the shape a paused actor has: the latest -// snapshot is node-local, whether or not atelet has uploaded it. A latest -// snapshot with only a durable copy, or a local one not captured yet, does not -// count. +// TestLatestSnapshotIsLocal pins when the actor's latest snapshot is +// node-local, whether or not atelet has uploaded it. A latest snapshot with +// only a durable copy, or a local one not captured yet, does not count. func TestLatestSnapshotIsLocal(t *testing.T) { local := func(gen int32) *ateapipb.Snapshot { return newPausedSnapshot(t, gen, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local", "node-1") diff --git a/cmd/ateapi/internal/controlapi/snapshot_report.go b/cmd/ateapi/internal/controlapi/snapshot_report.go index 5f57ff26f4..4b5ef70f28 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report.go @@ -164,8 +164,8 @@ func applyAteletResponseState(ctx context.Context, status *ateapipb.ActorStatus, // applyLatestDurable marks the reported durable snapshot COMPLETED, and records // its fidelity. Its uuid and upload URI must match the recorded ones, and its -// fidelity must be set. The snapshot keeps its locality, so a paused Actor -// stays on its node. +// fidelity must be set. The snapshot keeps its locality, so an Actor with a +// node-local snapshot stays on its node. // // A late report changes nothing and is logged at Info. A report is late when // its generation is older than the latest COMPLETED one, or is no longer in diff --git a/cmd/ateapi/internal/controlapi/snapshot_report_test.go b/cmd/ateapi/internal/controlapi/snapshot_report_test.go index 18e68386a3..435244ff7c 100644 --- a/cmd/ateapi/internal/controlapi/snapshot_report_test.go +++ b/cmd/ateapi/internal/controlapi/snapshot_report_test.go @@ -564,8 +564,8 @@ func TestApplyAteletResponseState(t *testing.T) { otherActorUID = "1a7e4c83-6d20-4f95-b3c8-9e0a2f6d4b17" ) - // previousPause is a pausing actor's status, holding the snapshot of the - // pause before, whose upload is pending. + // previousPause is a suspending actor's status, holding the snapshot of the + // async suspend before, whose upload is pending. previousPause := func() *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: suspending, AssignedNode: reportNode, LastAssignedGeneration: 2, Snapshots: []*ateapipb.Snapshot{ withDurable(t, 1, reportNode, pending), diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index 7f230c0e8f..36d2eb8783 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -1264,7 +1264,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { snap.SandboxRuntime = rt return snap } - // pauseUploadURI is where atelet uploads the pause snapshot. + // pauseUploadURI is where atelet uploads the local snapshot. pauseUploadURI := someActorSnapshotURI(t, testStorageLocation, "team-a", localSnapshotName) uploadedLocalSnap := func(name string, fidelity ateapipb.SnapshotFidelity) *ateapipb.Snapshot { snap := localSnap(name, fidelity) @@ -1284,7 +1284,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { // actorSeed is the actor status a row persists before resuming. type actorSeed struct { - // localSnapshot seeds a completed local snapshot (the pause checkpoint); + // localSnapshot seeds a completed local snapshot; // a non-nil value also pins the actor to the snapshot's node. localSnapshot *ateapipb.Snapshot // externalSnapshot seeds a completed durable snapshot. @@ -1493,8 +1493,8 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { want: restoreWant{code: codes.DataLoss}, }, { - // A pause snapshot's upload is PENDING until atelet reports it, - // but a local restore names it anyway, so atelet can download the + // An async suspend snapshot's upload is PENDING until atelet reports + // it, but a local restore names it anyway, so atelet can download the // snapshot if it detached the actor before the report landed. name: "13 Full pause snapshot restores locally as Full", actor: actorSeed{ @@ -1591,7 +1591,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { // An older external snapshot's template mismatch does not affect a - // local pause restore captured under the current template. + // local restore captured under the current template. name: "18 local snapshot ignores an older external snapshot's template mismatch", actor: actorSeed{ localSnapshot: func() *ateapipb.Snapshot { @@ -1741,7 +1741,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, { // A recorded upload changes nothing: a local restore names the - // pause snapshot's upload whether or not it is COMPLETED. + // local snapshot's upload whether or not it is COMPLETED. name: "25 uploaded pause snapshot restores locally and sends its upload", actor: actorSeed{ localSnapshot: uploadedLocalSnap(localSnapshotName, fullScope), @@ -1754,7 +1754,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { }, }, { - // A local restore names the pause snapshot's upload, so a record + // A local restore names the local snapshot's upload, so a record // that cannot name it fails, as a malformed durable snapshot URI // does. name: "26 local snapshot with no upload recorded fails with DataLoss", @@ -1807,7 +1807,7 @@ func TestResumeActor_AteletWireRequest(t *testing.T) { t.Fatal("created template has no UID; the matching tmplUID case would be vacuous") } - // Paused and suspended actors are both SUSPENDED; a pause + // Sync and async suspended actors are both SUSPENDED; a node-local // checkpoint is what makes the resume local. actorRef := resources.ActorRef{Atespace: "team-a", Name: "id1"} var extGen, localGen int32 @@ -2578,8 +2578,8 @@ func seedMicroVMWorker(t *testing.T, ctx context.Context, st store.Interface, po } // seedActorWithSnapshot stores actor team-a/id1 in state holding snap as its -// only snapshot. node is the AssignedNode a PAUSED actor keeps, which pins -// its resume to that node. +// only snapshot. node is the AssignedNode a suspended actor with a local +// snapshot keeps. func seedActorWithSnapshot(t *testing.T, ctx context.Context, st store.Interface, state ateapipb.ActorState, node string, snap *ateapipb.Snapshot) *ateapipb.Actor { t.Helper() return storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ @@ -2720,7 +2720,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing ctx := context.Background() st := newTestPersistence(t) seedRuntimeWorker(t, ctx, st, "node1-incompatible", "node-1", arm64RT) - // A paused actor: SUSPENDED, pinned to its node by a local snapshot. + // A suspended actor with a local snapshot on node-1. local := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "local-1", "node-1") local.SandboxRuntime = amd64RT actor := seedActorWithSnapshot(t, ctx, st, suspended, "node-1", local) @@ -2729,7 +2729,7 @@ func TestAssignWorkerAttempt_PrefersRuntimeMatchingWorkerAndFallsBack(t *testing if got := apierror.Code(err); got != codes.ResourceExhausted { t.Fatalf("apierror.Code(err) = %v, want %v (err: %v)", got, codes.ResourceExhausted, err) } - // Nothing was written: the actor waits for its node, still paused, + // Nothing was written: the actor waits for its node, still SUSPENDED, // unassigned, and holding no claim anything would have to release. after, err := st.GetActor(ctx, actorRef) if err != nil { diff --git a/cmd/ateapi/internal/controlapi/workflow_revert.go b/cmd/ateapi/internal/controlapi/workflow_revert.go index e851e3328d..55c239fe25 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert.go @@ -32,10 +32,10 @@ import ( ) // revertableStates are the states a revert is accepted from. A SUSPENDED -// actor is also accepted when it is paused, that is, when its latest snapshot -// is node-local: the revert discards the node-local copy and returns the actor -// to its latest durable snapshot, which is that same snapshot once atelet's -// upload of it is recorded. See isRevertable. +// actor is also accepted when its latest snapshot is node-local: the revert +// discards the node-local copy and returns the actor to its latest durable +// snapshot, which is that same snapshot once atelet's upload of it is +// recorded. See isRevertable. var revertableStates = []ateapipb.ActorState{ ateapipb.ActorState_ACTOR_STATE_RUNNING, ateapipb.ActorState_ACTOR_STATE_CRASHED, diff --git a/cmd/ateapi/internal/controlapi/workflow_revert_test.go b/cmd/ateapi/internal/controlapi/workflow_revert_test.go index 6a346166da..5405ee2d08 100644 --- a/cmd/ateapi/internal/controlapi/workflow_revert_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_revert_test.go @@ -37,7 +37,7 @@ import ( // SUSPENDED is the load-bearing rejection: it is revert's own destination, and // also what a suspend that won the race leaves behind, so accepting it would // report success for the opposite of what was asked. A SUSPENDED actor that -// holds a node-local snapshot is paused, not reverted, and is accepted; see +// holds a node-local snapshot is accepted; see // TestEnsureMarkedReverting_AcceptsPaused. func TestEnsureMarkedReverting_StateMatrix(t *testing.T) { allowed := map[ateapipb.ActorState]bool{ @@ -71,9 +71,9 @@ func TestEnsureMarkedReverting_StateMatrix(t *testing.T) { } } -// TestEnsureMarkedReverting_AcceptsPaused verifies a paused actor (SUSPENDED -// whose latest snapshot is node-local) takes the revert edge: the revert -// discards that snapshot and returns the actor to its durable one. +// TestEnsureMarkedReverting_AcceptsPaused verifies a SUSPENDED actor whose +// latest snapshot is node-local takes the revert edge: the revert discards that +// local copy and returns the actor to its durable snapshot. func TestEnsureMarkedReverting_AcceptsPaused(t *testing.T) { ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) @@ -107,7 +107,7 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { for _, tc := range []struct { name string seedState ateapipb.ActorState - // paused seeds the paused shape: the latest snapshot is node-local + // paused seeds the local-snapshot shape: the latest snapshot is node-local // and the actor is pinned to its node. Only meaningful with // SUSPENDED, which the revert otherwise rejects. paused bool @@ -133,9 +133,10 @@ func TestRevertActor_ReturnsActorToItsSnapshot(t *testing.T) { } // The snapshot revert must preserve, plus the node-local state a - // pause left behind, which it must not. A paused actor's latest - // snapshot is the completed local one; the other origins also carry - // the half-written local snapshot an interrupted pause leaves. + // suspend left behind, which it must not. The suspended actor's + // latest snapshot is the completed local one; the other origins + // also carry the half-written local snapshot an interrupted suspend + // leaves. const keptURI = "gs://snapshots/team-a/actors/keep/snapshot" tmpl := &ateapipb.ActorTemplate{SnapshotConfig: &ateapipb.SnapshotConfig{StorageLocation: "gs://snapshots"}} mustUpdateActorStatus(t, ctx, st, actor, func(s *ateapipb.ActorStatus) { @@ -335,11 +336,11 @@ func TestRevertActor_UnreachableNode(t *testing.T) { } } -// TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a paused -// actor whose snapshot atelet uploaded returns it to that upload, its latest -// durable snapshot, and collects the durable snapshot the upload replaced. The -// actor has no assigned node, so there is nothing to terminate, as in -// TestRevertActor_ReturnsActorToItsSnapshot. +// TestRevertActor_PausedSnapshotAlreadyUploaded verifies a revert of a +// suspended actor whose local snapshot atelet uploaded returns it to that +// upload, its latest durable snapshot, and collects the durable snapshot the +// upload replaced. The actor has no assigned node, so there is nothing to +// terminate, as in TestRevertActor_ReturnsActorToItsSnapshot. func TestRevertActor_PausedSnapshotAlreadyUploaded(t *testing.T) { memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY completed := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED @@ -495,8 +496,8 @@ func TestEnsureInProgressSnapshotDiscarded(t *testing.T) { // foreignInFlight records the in-progress snapshot under another // owner's prefix, which no suspend can produce. foreignInFlight bool - // earlierPause records an older PENDING snapshot: a pause whose upload - // atelet never reported before the actor resumed. + // earlierPause records an older PENDING snapshot: an async suspend whose + // upload atelet never reported before the actor resumed. earlierPause bool wantInFlightDiscarded bool wantEarlierPauseDiscarded bool diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 49e596d1fe..618e7bd68d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -35,18 +35,19 @@ import ( ) // suspendMode selects which of the two suspends the suspend workflow runs. -// Both take the same steps; the mode only decides the shape of the snapshot -// they take. +// Both take the same steps; the mode only decides when atelet uploads the +// durable copy of the snapshot they take. type suspendMode int const ( // suspendSync makes the snapshot durable before the workflow returns: - // the actor ends SUSPENDED holding a durable snapshot. + // the actor ends SUSPENDED holding a COMPLETED durable snapshot (plus its + // node-local copy when atelet retains it on the worker's node). suspendSync suspendMode = iota - // suspendAsync leaves the snapshot on the worker's node: the actor ends + // suspendAsync uploads the snapshot in the background: the actor ends // SUSPENDED holding a node-local snapshot whose durable copy is still - // pending, pinned to that node until atelet uploads it in the background - // and the detach report unpins it. + // PENDING, pinned to that node until atelet finishes the upload and the + // detach report unpins it. suspendAsync ) @@ -69,8 +70,8 @@ func (m suspendMode) uploadMode() ateletpb.UploadMode { // SuspendActor executes the workflow to suspend an actor. A running actor is // checkpointed on its worker: a sync suspend uploads the snapshot before -// returning, an async suspend keeps it on that node. A SUSPENDED actor is -// returned as it is, whatever the mode. Idempotent: a re-entered workflow +// returning, an async suspend uploads it in the background. A SUSPENDED actor +// is returned as it is, whatever the mode. Idempotent: a re-entered workflow // fast-forwards past the steps a previous attempt completed, deriving // progress from the persisted actor alone. func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef, mode suspendMode) (_ *ateapipb.Actor, err error) { @@ -211,9 +212,9 @@ func (w *ActorWorkflow) ensureMarkedSuspending(ctx context.Context, actorRef res // ensureAteletSuspended checkpoints the workload to the actor's persisted // in-progress snapshot location. A sync suspend has atelet upload it before -// returning; an async suspend has atelet keep it on the node. It returns the -// snapshot state atelet reports once the checkpoint is taken, which finalize -// applies. +// returning; an async suspend has atelet upload it in the background. It +// returns the snapshot state atelet reports once the checkpoint is taken, +// which finalize applies. // A re-entered workflow re-sends the same snapshot, which // ensureMarkedSuspending minted once, and atelet answers a Checkpoint of a // snapshot it already committed without checkpointing again, so a retry after @@ -363,7 +364,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r } // Both modes mark the durable copy PENDING up front. Without it a sync // suspend has nothing to mark COMPLETED, and atelet's background upload of - // a pause's snapshot could never be recorded. + // an async suspend's snapshot could never be recorded. if snapshotAtLatestGeneration(latestActor.GetStatus()).GetDurableSnapshot() == nil { return nil, crashActorAndFail(ctx, w.store, actorRef, ateattr.OperationSuspend, crashMessageDurableSnapshotMissing) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go index b0d0a8ed98..8e19b7717d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_async_test.go @@ -30,13 +30,13 @@ import ( ) // TestSuspendActorAsync_Finalized_WorkerGone reproduces the scenario where the -// worker pod disappears from the DB during pause finalization, so the node it -// ran on is unknown. +// worker pod disappears from the DB during async suspend finalization, so the +// node it ran on is unknown. // // Current behavior: AssignedNode is left empty, and the actor is crashed -// instead of left paused, since a local snapshot with an unknown node can -// never be safely resumed. The pause fails with FailedPrecondition rather than -// reporting the pause as done. +// instead of left SUSPENDED with a local snapshot, since a local snapshot with +// an unknown node can never be safely resumed. The suspend fails with +// FailedPrecondition rather than reporting the suspend as done. func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) @@ -84,7 +84,7 @@ func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { } // This site records the crash counter, so it must write the record too, or - // a pause-finalize crash is the one kind nothing can attribute to an actor. + // an async-suspend-finalize crash is the one kind nothing can attribute to an actor. if len(*records) != 1 { t.Fatalf("got %d crash records, want 1", len(*records)) } @@ -98,8 +98,8 @@ func TestSuspendActorAsync_Finalized_WorkerGone(t *testing.T) { // TestSuspendActorAsync_Finalized_AlreadyCrashed verifies that if the actor was // crashed out-of-band while it checkpointed, and has no AssignedNode, the -// pause's finalize preserves the existing Crash status and snapshots, emits no -// duplicate crash log record, and still fails. +// async suspend's finalize preserves the existing Crash status and snapshots, +// emits no duplicate crash log record, and still fails. func TestSuspendActorAsync_Finalized_AlreadyCrashed(t *testing.T) { ctx := context.Background() st := newTestPersistence(t) @@ -149,7 +149,7 @@ func TestSuspendActorAsync_Finalized_AlreadyCrashed(t *testing.T) { } // TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete verifies that a crash the -// DeleteWorker workflow commits while the pause's finalize is freeing the +// DeleteWorker workflow commits while the async suspend's finalize is freeing the // worker is kept: the finalize update is pinned to the actor version it // checked, so it fails Aborted instead of committing SUSPENDED over the crash. func TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { @@ -174,7 +174,7 @@ func TestSuspendActorAsync_Finalized_KeepsCrashFromWorkerDelete(t *testing.T) { assertConcurrentCrashKept(t, ctx, st, actorRef, err, "suspend failed: "+crashMessageWorkerPodGone) } -// TestSuspendActorAsync_Finalized_RecordsFidelity verifies pause records the template's +// TestSuspendActorAsync_Finalized_RecordsFidelity verifies async suspend records the template's // preferred_fidelity on the local Snapshot alongside Locality, and keeps its // durable copy PENDING for atelet to upload in the background. func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { @@ -262,9 +262,9 @@ func TestSuspendActorAsync_Finalized_RecordsFidelity(t *testing.T) { // TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment verifies that if the // latest generation snapshot entry, or its PENDING durable copy, is missing -// when the pause finalizes, the actor transitions to CRASHED and clears +// when the async suspend finalizes, the actor transitions to CRASHED and clears // WorkerAssignment after releasing the worker rather than getting stuck -// SUSPENDING, and the pause fails. +// SUSPENDING, and the suspend fails. func TestSuspendActorAsync_Finalized_MissingSnapshotClearsAssignment(t *testing.T) { tests := []struct { name string @@ -399,9 +399,9 @@ func TestSuspendActorAsync_AteletPaused_DialFailureLeavesActorRetryable(t *testi } // TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod verifies that a -// pausing actor with no worker pod recorded is moved to CRASHED by the -// corrupted-assignment check before atelet is called, and the pause fails -// with FailedPrecondition. +// suspending actor with no worker pod recorded is moved to CRASHED by the +// corrupted-assignment check before atelet is called, and the async suspend +// fails with FailedPrecondition. func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) { ctx := context.Background() st, cleanup := storetest.SetupTestStore(t) @@ -427,9 +427,9 @@ func TestSuspendActorAsync_CrashesWhenPausingActorMissingWorkerPod(t *testing.T) } } -// TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that local pause -// checkpoints forward the template's PreferredFidelity to atelet, and label the -// pause's lifecycle-op metric with it. +// TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity verifies that async +// suspend checkpoints forward the template's PreferredFidelity to atelet, and +// label the suspend's lifecycle-op metric with it. func TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity(t *testing.T) { tests := []struct { preferred ateapipb.SnapshotFidelity @@ -488,7 +488,7 @@ func TestSuspendActorAsync_AteletPaused_ForwardsPreferredFidelity(t *testing.T) if got := req.GetUploadConfig().GetMode(); got != ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { t.Errorf("CheckpointRequest.UploadConfig.Mode = %v, want UPLOAD_MODE_DEFERRED", got) } - // The fidelity sent to atelet labels the pause's lifecycle-op metric. + // The fidelity sent to atelet labels the suspend's lifecycle-op metric. assertLifecycleFidelity(t, reader, ateattr.SnapshotFidelityValue(tc.want)) }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index 304a1b0c3d..4110374646 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -612,10 +612,11 @@ func TestSuspendActor_Finalized_ReleasesReplacedSnapshot(t *testing.T) { } } -// TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot verifies that a -// pause keeping the previous pause's upload collects the external snapshot -// that upload replaced, and keeps the upload. The upload is recorded from the -// pause's Checkpoint state, or before the pause from atelet's report. +// TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot verifies that an +// async suspend keeping the previous async suspend's upload collects the +// external snapshot that upload replaced, and keeps the upload. The upload is +// recorded from the Checkpoint state, or before the suspend from atelet's +// report. func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { const node = "node-1" memory := ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY @@ -623,8 +624,8 @@ func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { pending := ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING for _, tc := range []struct { name string - // reported records the upload before the pause, as atelet's report - // does. Otherwise only the pause's Checkpoint state carries it. + // reported records the upload before the suspend, as atelet's report + // does. Otherwise only the suspend's Checkpoint state carries it. reported bool }{ {name: "recorded from the pause's Checkpoint state"}, @@ -647,8 +648,9 @@ func TestSuspendActor_Finalized_PauseReleasesReplacedSnapshot(t *testing.T) { }, }) - // The actor suspended to old, paused to snap-2 and resumed from - // it, and is now pausing to snap-3. atelet has uploaded snap-2. + // The actor suspended to old, suspended asynchronously to snap-2 and + // resumed from it, and is now suspending asynchronously to snap-3. + // atelet has uploaded snap-2. old := mustActorSnapshotURI(t, template, actor, "old") paused := mustActorSnapshotURI(t, template, actor, "snap-2") pausing := mustActorSnapshotURI(t, template, actor, "snap-3") @@ -1445,9 +1447,9 @@ func TestSuspendActor_AppliesCheckpointState(t *testing.T) { }, wantSnapshots: []snapshotOutcome{{generation: 1, fidelity: memory, durable: completed}, {generation: 2, fidelity: memory, locality: node, durable: pending}}, }, { - // A RUNNING actor resumed from a pause keeps that pause's snapshot, - // with its PENDING durable copy. atelet uploads it in the background, - // and the next pause records that. + // A RUNNING actor resumed from a local snapshot keeps that async + // suspend's snapshot, with its PENDING durable copy. atelet uploads it + // in the background, and the next async suspend records that. name: "async: the previous pause's upload atelet finished is recorded as COMPLETED", mode: suspendAsync, status: &ateapipb.ActorStatus{LastAssignedGeneration: 1, Snapshots: []*ateapipb.Snapshot{newPausedSnapshot(t, 1, memory, "", "snap-1", node)}}, diff --git a/cmd/ateapi/internal/controlapi/workflow_tag.go b/cmd/ateapi/internal/controlapi/workflow_tag.go index fa18332620..504e70aee7 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag.go @@ -37,8 +37,8 @@ import ( // // The tag is built in 6 phases: // 1. Check that the name is free and the storage location is usable. -// 2. Make the actor's latest snapshot durable, if it is not uploaded yet: a -// paused actor's snapshot may still be uploading. +// 2. Make the actor's latest snapshot durable, if it is not uploaded yet: an +// asynchronously suspended actor's snapshot may still be uploading. // 3. Reserve the tag's name; the store assigns its UID. // 4. Record the snapshot's URI, under the tag's UID, before anything is // copied there. @@ -269,11 +269,11 @@ func (w *ActorWorkflow) checkTagCreatable(ctx context.Context, tagRef resources. } // ensureTagSourceSnapshotDurable makes the actor's latest snapshot durable when -// it is not uploaded yet, the shape a pause leaves until atelet's background -// upload is recorded. It asks the atelet on the snapshot's node to upload it -// with PromoteSnapshot, which keeps the local copy so the actor still resumes -// on that node, and records the upload on the actor. It returns the actor as -// stored afterwards. +// it is not uploaded yet, the shape an asynchronous suspend leaves until +// atelet's background upload is recorded. It asks the atelet on the snapshot's +// node to upload it with PromoteSnapshot, which keeps the local copy so the +// actor still resumes on that node, and records the upload on the actor. It +// returns the actor as stored afterwards. // // A failed upload does not crash the actor: the tag fails and the actor is // left as it was. diff --git a/cmd/ateapi/internal/controlapi/workflow_tag_test.go b/cmd/ateapi/internal/controlapi/workflow_tag_test.go index 8820c30890..557ffd878f 100644 --- a/cmd/ateapi/internal/controlapi/workflow_tag_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_tag_test.go @@ -241,8 +241,8 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { // snapshot is always at generation extGen. lastAssigned int32 extGen int32 - // locality is set on the external snapshot, as when a paused actor's - // local snapshot was uploaded in place. + // locality is set on the external snapshot, as when the snapshot also + // has a local copy on the node. locality string }{ { @@ -287,11 +287,12 @@ func TestTagActorSnapshot_TagsNewestExternalSnapshot(t *testing.T) { } } -// TestTagActorSnapshot_PromotesLocalSnapshot verifies that tagging a paused -// actor whose snapshot is not uploaded yet has the atelet on the snapshot's -// node upload it first, records the upload on the actor, and tags that -// snapshot. A failed upload fails the tag without reserving its name and leaves -// the actor as it was. A create that cannot succeed fails before the upload. +// TestTagActorSnapshot_PromotesLocalSnapshot verifies that tagging a suspended +// actor whose local snapshot is not uploaded yet has the atelet on the +// snapshot's node upload it first, records the upload on the actor, and tags +// that snapshot. A failed upload fails the tag without reserving its name and +// leaves the actor as it was. A create that cannot succeed fails before the +// upload. func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { // reported is the state atelet answers with once it made snap durable. reported := func(snap *ateapipb.Snapshot) *ateletpb.ActorSnapshotState { @@ -306,22 +307,22 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { tests := []struct { name string - // olderDurable seeds a COMPLETED snapshot below the paused one. + // olderDurable seeds a COMPLETED snapshot below the local one. olderDurable bool - // uploaded records the paused snapshot's durable copy COMPLETED. + // uploaded records the local snapshot's durable copy COMPLETED. uploaded bool - // nodeLost clears the paused snapshot's locality, as when its node + // nodeLost clears the local snapshot's locality, as when its node // was lost before the upload finished. nodeLost bool // nameTaken seeds a tag under the name the create asks for. nameTaken bool // badLocation points the template at an unusable storage location. badLocation bool - // noTemplateUID records no template the paused snapshot was built + // noTemplateUID records no template the local snapshot was built // under, a broken row. noTemplateUID bool // promoteState builds the state PromoteSnapshot answers with from the - // paused snapshot. + // local snapshot. promoteState func(paused *ateapipb.Snapshot) *ateletpb.ActorSnapshotState promoteErr error wantPromote bool @@ -566,13 +567,13 @@ func TestTagActorSnapshot_PromotesLocalSnapshot(t *testing.T) { return } - // The paused snapshot is now durable and stays on its node. + // The local snapshot is now durable and stays on its node. wantPaused := proto.CloneOf(paused) wantPaused.DurableSnapshot.Status = ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_COMPLETED if diff := cmp.Diff(wantPaused, findSnapshotByGeneration(stored.GetStatus().GetSnapshots(), 2), protocmp.Transform()); diff != "" { t.Errorf("the actor's latest snapshot mismatch (-want +got):\n%s", diff) } - // The tag copied the paused snapshot, not the older one. + // The tag copied the local snapshot, not the older one. if diff := cmp.Diff([]string{"manifest.json", "memory.zst"}, objects.Snapshot(t, mustParseSnapshotURI(t, tagSnapshot))); diff != "" { t.Errorf("the tag's external snapshot mismatch (-want +got):\n%s", diff) } diff --git a/cmd/ateapi/internal/controlapi/workflow_test.go b/cmd/ateapi/internal/controlapi/workflow_test.go index fe24540cbd..4d0bbb6c36 100644 --- a/cmd/ateapi/internal/controlapi/workflow_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_test.go @@ -168,9 +168,9 @@ func TestActorStateChangeRecords(t *testing.T) { } } -// markOnly runs a suspend or pause on a RUNNING actor only as far as its mark -// step: the actor gets a worker to checkpoint on, but the atelet dial fails, -// so the SUSPENDING mark is the only transition the workflow commits. +// markOnly runs a suspend (sync or async) on a RUNNING actor only as far as its +// mark step: the actor gets a worker to checkpoint on, but the atelet dial +// fails, so the SUSPENDING mark is the only transition the workflow commits. func markOnly(t *testing.T, w *ActorWorkflow, ref resources.ActorRef, actor *ateapipb.Actor, mode suspendMode) { t.Helper() ctx := context.Background() diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index c4e7af220f..2b347393bb 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -157,9 +157,10 @@ func seedWorkflowActor(t *testing.T, ctx context.Context, st store.Interface, ac storetest.MustCreateActor(t, ctx, st, actor) } -// withPausedShape seeds the shape a pause leaves behind: a node-local -// snapshot at the latest generation with a PENDING durable copy, pinned to -// node. Pair it with ACTOR_STATE_SUSPENDED to seed a paused actor. +// withPausedShape seeds the shape an asynchronous suspend leaves behind: a +// node-local snapshot at the latest generation with a PENDING durable copy, +// pinned to node. Pair it with ACTOR_STATE_SUSPENDED to seed an asynchronously +// suspended actor. func withPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { t.Helper() snap := newPausedSnapshot(t, 1, ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_MEMORY, "", "pause-snap", node) @@ -170,8 +171,9 @@ func withPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { } } -// withUploadedPausedShape seeds a paused actor whose snapshot atelet has since -// uploaded: the paused shape, with its durable copy COMPLETED. +// withUploadedPausedShape seeds an asynchronously suspended actor whose +// snapshot atelet has since uploaded: the local-snapshot shape, with its +// durable copy COMPLETED. func withUploadedPausedShape(t *testing.T, node string) func(*ateapipb.Actor) { t.Helper() paused := withPausedShape(t, node) @@ -189,9 +191,10 @@ func newInProgressSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFi return newDurableSnapshot(gen, ateapipb.SnapshotOwner_SNAPSHOT_OWNER_ACTOR, fidelity, "", uuid, someActorSnapshotURI(t, testStorageLocation, "team-a", uuid), ateapipb.SnapshotStorageStatus_SNAPSHOT_STORAGE_STATUS_PENDING) } -// newPausedSnapshot builds the snapshot a pause leaves on node locality at -// generation gen: a node-local checkpoint whose PENDING durable copy atelet -// uploads in the background. With no locality it is the in-progress mark. +// newPausedSnapshot builds the snapshot an asynchronous suspend leaves on node +// locality at generation gen: a node-local checkpoint whose PENDING durable +// copy atelet uploads in the background. With no locality it is the in-progress +// mark. func newPausedSnapshot(t *testing.T, gen int32, fidelity ateapipb.SnapshotFidelity, templateUID, uuid, locality string) *ateapipb.Snapshot { t.Helper() snap := newInProgressSnapshot(t, gen, fidelity, uuid) @@ -252,9 +255,8 @@ func suspendTestTemplateRef() *ateapipb.ObjectRef { return &ateapipb.ObjectRef{Atespace: "ns", Name: "tmpl1"} } -// seedSuspendTemplate stores the ActorTemplate the suspend and pause workflow -// tests run their actors from, with the given snapshot config, and returns it -// as stored. +// seedSuspendTemplate stores the ActorTemplate the suspend workflow tests run +// their actors from, with the given snapshot config, and returns it as stored. func seedSuspendTemplate(t *testing.T, ctx context.Context, st store.Interface, snapshotConfig *ateapipb.SnapshotConfig) *ateapipb.ActorTemplate { t.Helper() ref := suspendTestTemplateRef() @@ -310,8 +312,8 @@ func assignmentOf(actor *ateapipb.Actor) *ateapipb.ActorAssignment { } // newMarkOnlyWorkflow builds an ActorWorkflow whose atelet dial always fails, -// so a suspend or pause stops right after its mark step and leaves the actor -// as that step committed it. +// so a suspend stops right after its mark step and leaves the actor as that +// step committed it. func newMarkOnlyWorkflow(st store.Interface) *ActorWorkflow { return &ActorWorkflow{store: st, dialer: newDanglingDialer()} } diff --git a/cmd/ateapi/internal/workerservice/certificate.go b/cmd/ateapi/internal/workerservice/certificate.go index 7d75515811..e0433346aa 100644 --- a/cmd/ateapi/internal/workerservice/certificate.go +++ b/cmd/ateapi/internal/workerservice/certificate.go @@ -56,16 +56,17 @@ func (s *Server) MintWorkerActorCertificate(ctx context.Context, req *ateapipb.M return nil, fmt.Errorf("while retrieving actor: %w", err) } // Only the assigned worker may mint, via its node's atelet, checked before the - // UID so other callers get NotFound. Unlike MintWorkerActorJwt, a paused actor - // gets NotFound too: it has no worker to bind the certificate to. + // UID so other callers get NotFound. Unlike MintWorkerActorJwt, a SUSPENDED + // actor gets NotFound too: it has no worker to bind the certificate to. if err := s.checkActorOnNode(ctx, dbActor, caller.NodeName, "certificate"); err != nil { return nil, err } // The status must name the requesting worker. When it names one, // checkActorOnNode has confirmed in the store's worker assignments that - // this worker holds the actor. When it names none (a paused actor, or a - // resume that has bound a worker but not yet recorded it), checkActorOnNode - // read assigned_node alone, and no worker runs the actor yet. + // this worker holds the actor. When it names none (a SUSPENDED actor on + // assigned_node, or a resume that has bound a worker but not yet recorded + // it), checkActorOnNode read assigned_node alone, and no worker runs the + // actor yet. if assigned := dbActor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); assigned == "" || assigned != req.GetWorker().GetName() { slog.WarnContext(ctx, "Refusing a worker-for-actor certificate for an actor not assigned to the calling worker", slog.String("actor", resources.ActorRefFromActor(dbActor).String()), diff --git a/cmd/ateapi/internal/workerservice/certificate_test.go b/cmd/ateapi/internal/workerservice/certificate_test.go index 76f98a871e..a112c964ae 100644 --- a/cmd/ateapi/internal/workerservice/certificate_test.go +++ b/cmd/ateapi/internal/workerservice/certificate_test.go @@ -202,8 +202,8 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { onWorker := func(state ateapipb.ActorState) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: state, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}} } - // pausingOnWorker is a pause in progress: SUSPENDING on worker-1 with a - // latest snapshot that is node-local and not yet captured. + // pausingOnWorker is an async suspend in progress: SUSPENDING on worker-1 + // with a latest snapshot that is not yet captured. pausingOnWorker := func() *ateapipb.ActorStatus { status := onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING) status.AssignedNode = "node-1" @@ -254,7 +254,7 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.OK, }, { - // Pause, suspend and crash release the worker before they clear the + // Suspend and crash release the worker before they clear the // actor's status, which keeps naming the worker until they do. name: "running actor whose worker was released", ctx: authed, @@ -364,8 +364,8 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.NotFound, }, { - // MintWorkerActorJwt serves a paused actor's checkpoint node. No - // worker runs a paused actor, so no worker gets its certificate. + // MintWorkerActorJwt serves a node holding an actor's local snapshot. No + // worker runs a SUSPENDED actor, so no worker gets its certificate. name: "paused actor on this node", ctx: authed, status: pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, "node-1"), @@ -399,7 +399,7 @@ func TestMintWorkerActorCertificate_Errors(t *testing.T) { want: codes.NotFound, }, { - // An empty worker name must not match a paused actor's empty + // An empty worker name must not match a SUSPENDED actor's empty // assignment. name: "paused actor, empty worker name", ctx: authed, diff --git a/cmd/ateapi/internal/workerservice/jwt.go b/cmd/ateapi/internal/workerservice/jwt.go index 977fbbcb7c..ca678eddeb 100644 --- a/cmd/ateapi/internal/workerservice/jwt.go +++ b/cmd/ateapi/internal/workerservice/jwt.go @@ -49,7 +49,7 @@ func (s *Server) MintWorkerActorJwt(ctx context.Context, req *ateapipb.MintWorke } // The actor doesn't need to be running: tokens are needed to fetch a - // snapshot before resume and to upload a checkpoint after pause. + // snapshot before resume and to upload a checkpoint during or after suspend. dbActor, err := s.store.GetActor(ctx, resources.ActorRefFromObjectRef(req.GetActor())) if errors.Is(err, store.ErrNotFound) { return nil, apierror.NotFound("actor not found") diff --git a/cmd/ateapi/internal/workerservice/jwt_test.go b/cmd/ateapi/internal/workerservice/jwt_test.go index 66f2779bd1..40ea33db01 100644 --- a/cmd/ateapi/internal/workerservice/jwt_test.go +++ b/cmd/ateapi/internal/workerservice/jwt_test.go @@ -113,9 +113,10 @@ func TestMintWorkerActorJwt(t *testing.T) { } } -// pausedStatus is how a paused actor looks: SUSPENDED, no worker, assigned_node -// on the node holding its local snapshot, and a latest generation that is -// local-only. The latest generation is the one last_assigned_generation names. +// pausedStatus is how an asynchronously suspended actor looks: SUSPENDED, no +// worker, assigned_node on the node holding its local snapshot, and a latest +// generation that is local-only. The latest generation is the one +// last_assigned_generation names. func pausedStatus(state ateapipb.ActorState, node string) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{ State: state, @@ -125,9 +126,9 @@ func pausedStatus(state ateapipb.ActorState, node string) *ateapipb.ActorStatus } } -// uploadPendingStatus is how a paused actor looks until atelet's background -// upload of its local snapshot is recorded: SUSPENDED, no worker, and the -// latest generation's durable copy PENDING. +// uploadPendingStatus is how an asynchronously suspended actor looks until +// atelet's background upload of its local snapshot is recorded: SUSPENDED, no +// worker, and the latest generation's durable copy PENDING. func uploadPendingStatus(node string) *ateapipb.ActorStatus { status := pausedStatus(ateapipb.ActorState_ACTOR_STATE_SUSPENDED, node) status.Snapshots[0].DurableSnapshot = &ateapipb.SnapshotStorage{ @@ -154,8 +155,8 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { onWorker := func(state ateapipb.ActorState) *ateapipb.ActorStatus { return &ateapipb.ActorStatus{State: state, WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: "worker-1"}}} } - // pausingOnWorker is a pause in progress: SUSPENDING on worker-1 with a - // latest snapshot that is node-local and not yet captured. + // pausingOnWorker is an async suspend in progress: SUSPENDING on worker-1 + // with a latest snapshot that is not yet captured. pausingOnWorker := func() *ateapipb.ActorStatus { status := onWorker(ateapipb.ActorState_ACTOR_STATE_SUSPENDING) status.AssignedNode = "node-1" @@ -206,7 +207,7 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { want: codes.OK, }, { - // Pause, suspend and crash release the worker before they clear the + // Suspend and crash release the worker before they clear the // actor's status, which keeps naming the worker until they do. name: "running actor whose worker was released", ctx: authed, @@ -340,8 +341,8 @@ func TestMintWorkerActorJwt_Errors(t *testing.T) { want: codes.NotFound, }, { - // atelet uploads the paused actor's checkpoint in the background - // from the node that holds it, with no worker assigned. + // atelet uploads the asynchronously suspended actor's checkpoint in + // the background from the node that holds it, with no worker assigned. name: "paused actor with its upload pending on this node", ctx: authed, status: uploadPendingStatus("node-1"), diff --git a/cmd/ateapi/internal/workerservice/suspend.go b/cmd/ateapi/internal/workerservice/suspend.go index 6035fa29e2..4c58c44319 100644 --- a/cmd/ateapi/internal/workerservice/suspend.go +++ b/cmd/ateapi/internal/workerservice/suspend.go @@ -47,7 +47,7 @@ func (s *Server) RequestActorSuspend(ctx context.Context, req *ateapipb.RequestA // From here it is an ordinary suspend. Running the same workflow Control // runs is the point: its preconditions already refuse an Actor that is not - // suspendable, and a proposal that races a resume, pause, or delete loses + // suspendable, and a proposal that races a resume, suspend, or delete loses // to whichever holds the Actor's lease. // // TODO: the checks above run outside the Actor's lease, so there is a small @@ -119,7 +119,8 @@ func checkActorHostedBy(ctx context.Context, actor *ateapipb.Actor, workerName, return apierror.NotFound("Actor %s not found", actorRef) } // An Actor with no assignment has no worker to speak for it: it is - // SUSPENDED (paused or not) or CRASHED, and nothing hosts it. + // SUSPENDED (with or without a local snapshot) or CRASHED, and nothing + // hosts it. if assigned := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); assigned != workerName { slog.WarnContext(ctx, "Refusing a request for an actor hosted elsewhere", slog.String("worker", workerName), diff --git a/cmd/ateapi/internal/workerservice/suspend_test.go b/cmd/ateapi/internal/workerservice/suspend_test.go index 1599627c55..1a7425a642 100644 --- a/cmd/ateapi/internal/workerservice/suspend_test.go +++ b/cmd/ateapi/internal/workerservice/suspend_test.go @@ -209,7 +209,7 @@ func TestRequestActorSuspend_Errors(t *testing.T) { } // The suspend is a proposal, and the control plane's refusal is the answer: a -// request that loses a race to a resume, pause, or delete comes back as what +// request that loses a race to a resume, suspend, or delete comes back as what // the workflow said, not as something the worker has to interpret. func TestRequestActorSuspend_PassesThroughTheWorkflowsRefusal(t *testing.T) { st, cleanup := storetest.SetupTestStore(t) diff --git a/cmd/atelet/internal/ateletpath/ateletpath.go b/cmd/atelet/internal/ateletpath/ateletpath.go index 483d40a989..55d1b4450f 100644 --- a/cmd/atelet/internal/ateletpath/ateletpath.go +++ b/cmd/atelet/internal/ateletpath/ateletpath.go @@ -154,7 +154,7 @@ func LocalCheckpointsDir(actorUID string) string { ) } -// LocalSnapshotDir is the directory holding one named local (pause) snapshot +// LocalSnapshotDir is the directory holding one named node-local snapshot // of an actor: the checkpoint files plus their manifest. func LocalSnapshotDir(actorUID, snapshotName string) string { return filepath.Join(LocalCheckpointsDir(actorUID), snapshotName) diff --git a/cmd/atelet/internal/snapshottracker/actor.go b/cmd/atelet/internal/snapshottracker/actor.go index a85867c64c..648cfd837b 100644 --- a/cmd/atelet/internal/snapshottracker/actor.go +++ b/cmd/atelet/internal/snapshottracker/actor.go @@ -140,10 +140,10 @@ type actorSnapshotsRecord struct { // checkpoint has superseded it. // // Measuring from UndurableSince rather than LatestLocal.CommitTime - // ensures an actor that pauses and resumes repeatedly still uploads - // within uploadDelay instead of postponing durability indefinitely. An - // upload changes UndurableSince only once it succeeds, and a failed or - // interrupted one leaves it as it was. + // ensures an actor that suspends asynchronously and resumes repeatedly + // still uploads within uploadDelay instead of postponing durability + // indefinitely. An upload changes UndurableSince only once it succeeds, and + // a failed or interrupted one leaves it as it was. UndurableSince time.Time `json:"undurableSince,omitzero"` // DetachAt is when atelet next checks whether to detach the actor. Zero // means no detach is scheduled (see detachScheduled). @@ -550,7 +550,7 @@ func (a *actorSnapshots) detachScheduled() bool { // whether it is durable already. It fails with FailedPrecondition when no // detach can make it durable until a commit replaces it: LatestDurable names // the same snapshot differently, or LatestLocal is not durable and names no -// object storage location to upload it to, as after a pause today. +// object storage location to upload it to. func (a *actorSnapshots) detachTarget() (target SnapshotRef, alreadyDurable bool, err error) { target = *a.LatestLocal alreadyDurable, err = a.isLatestDurable(target) diff --git a/cmd/atelet/internal/snapshottracker/detach_test.go b/cmd/atelet/internal/snapshottracker/detach_test.go index a26b2784e8..20161dab5e 100644 --- a/cmd/atelet/internal/snapshottracker/detach_test.go +++ b/cmd/atelet/internal/snapshottracker/detach_test.go @@ -334,7 +334,7 @@ func TestTrackerDetach(t *testing.T) { } } -// A local snapshot no detach can make durable, as a pause takes today, never +// A local snapshot no detach can make durable (e.g. with no upload URI) never // comes due for a background detach, rather than failing it again and again. // A checkpoint that replaces it schedules a detach again. func TestTrackerSkipsUndetachableSnapshots(t *testing.T) { diff --git a/cmd/atelet/internal/snapshottracker/tracker_test.go b/cmd/atelet/internal/snapshottracker/tracker_test.go index 807e54e68d..30114f1e20 100644 --- a/cmd/atelet/internal/snapshottracker/tracker_test.go +++ b/cmd/atelet/internal/snapshottracker/tracker_test.go @@ -567,7 +567,7 @@ func TestTrackerCommitPromoted(t *testing.T) { } } -// A promote of a paused actor's local snapshot leaves its detach schedule as +// A promote of an actor's local snapshot leaves its detach schedule as // it is: the detach its checkpoint scheduled, or none for an actor whose Run // or Restore has begun since the snapshot was taken. func TestTrackerCommitPromotedKeepsDetachSchedule(t *testing.T) { diff --git a/cmd/atelet/internal/snapshottracker/upload_test.go b/cmd/atelet/internal/snapshottracker/upload_test.go index 2ba7eec135..863dd8dcfd 100644 --- a/cmd/atelet/internal/snapshottracker/upload_test.go +++ b/cmd/atelet/internal/snapshottracker/upload_test.go @@ -512,7 +512,7 @@ func TestTrackerWithoutStoreSchedulesNoUploads(t *testing.T) { } } -// A local snapshot no upload can make durable, as a pause takes today, never +// A local snapshot no upload can make durable (e.g. with no upload URI) never // comes due for a background upload, rather than failing it again and again. // A checkpoint that replaces it schedules an upload again. func TestTrackerSkipsUnuploadableSnapshots(t *testing.T) { @@ -803,9 +803,9 @@ func TestTrackerBeginPromote(t *testing.T) { } } -// When a background upload of s1 fails after a pause replaced LatestLocal with -// s2 (which has no object storage location), s1 stays in Uploading on disk and -// is retried. Further pause checkpoints keep s1 on disk and preserve its +// When a background upload of s1 fails after a checkpoint replaced LatestLocal +// with s2 (which has no object storage location), s1 stays in Uploading on disk +// and is retried. Further local checkpoints keep s1 on disk and preserve its // backoff, and once s1's retry succeeds, s1's local copy is deleted. func TestTrackerRetriesUploadingWhenLatestLocalHasNoObject(t *testing.T) { store := newFakeSnapshotStore() @@ -838,7 +838,7 @@ func TestTrackerRetriesUploadingWhenLatestLocalHasNoObject(t *testing.T) { t.Fatalf("state after failed upload = %v, want s2 local, s1 uploading, due at %v", st, retryAt) } - // Another pause checkpoint during backoff still keeps s1 and its backoff. + // Another local checkpoint during backoff still keeps s1 and its backoff. s3 := testRef("s3", 3, tr.clock.Now()) s3.Snapshot.Object = nil st = tr.CommitLocal(trackedActorUID, s3) @@ -1278,7 +1278,7 @@ func TestTrackerUploadActsForTheActor(t *testing.T) { if got := waitStarted(t, store); got != "s1" { t.Fatalf("before started = %q, want s1", got) } - // A pause snapshot no upload can make durable leaves s1 in Uploading, + // A local snapshot no upload can make durable leaves s1 in Uploading, // so the resumed upload reads its actor from there. s2 := testRef("s2", 2, t0.Add(time.Minute)) s2.Snapshot.Object = nil diff --git a/cmd/atelet/lifecycle_test.go b/cmd/atelet/lifecycle_test.go index a78bcf24af..1c6cbaccf8 100644 --- a/cmd/atelet/lifecycle_test.go +++ b/cmd/atelet/lifecycle_test.go @@ -205,11 +205,12 @@ func lifecycleSnapshot(actorUID, name string, generation int32, fidelity ateletp } // TestLocalSnapshotGC walks an actor through -// run -> pause -> resume -> pause -> terminate over atelet's RPC surface and -// ensures that superseded local snapshots are pruned by the next checkpoint -// and all remaining state is garbage collected after the actor is terminated. -// It runs at every fidelity atelet forwards, pinning that the pause and the -// resume hand worker the fidelity the control plane asked for, unchanged. +// run -> deferred checkpoint -> restore -> deferred checkpoint -> terminate +// over atelet's RPC surface and ensures that superseded local snapshots are +// pruned by the next checkpoint and all remaining state is garbage collected +// after the actor is terminated. It runs at every fidelity atelet forwards, +// pinning that the checkpoint and the restore hand worker the fidelity the +// control plane asked for, unchanged. func TestLocalSnapshotGC(t *testing.T) { for _, fidelity := range []ateletpb.SnapshotFidelity{ ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, @@ -270,7 +271,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir t.Fatalf("Run: %v", err) } - // Pause: a deferred checkpoint, which leaves the snapshot on this node. + // Async suspend: a deferred checkpoint, which leaves the snapshot on this node. resp, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: atespace, ActorName: actorName, @@ -289,8 +290,8 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir if _, err := os.Stat(snapshotFile); err != nil { t.Fatalf("pause did not write the local snapshot: %v", err) } - // A paused actor stays resumable on this node, so it keeps its staged - // assets, and with them its binaries in the cache. + // An actor with a local snapshot stays resumable on this node, so it keeps + // its staged assets, and with them its binaries in the cache. if _, err := os.Stat(stagedRunsc(actorUID)); err != nil { t.Errorf("paused actor lost its staged runsc: %v", err) } @@ -335,7 +336,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir t.Fatalf("expected RestoreStateDir to remain empty on local pause restore, got entries=%v err=%v", entries, err) } - // Second pause: must replace, not accumulate alongside, the first snapshot. + // Second deferred checkpoint: must replace, not accumulate alongside, the first snapshot. if _, err := s.Checkpoint(ctx, &ateletpb.CheckpointRequest{ Atespace: atespace, ActorName: actorName, @@ -371,7 +372,7 @@ func runLocalSnapshotGC(t *testing.T, fidelity ateletpb.SnapshotFidelity, useDir } // Every RPC hands worker the same directory set, except that a local - // restore points restore_dir at the pause snapshot; the fake already + // restore points restore_dir at the local snapshot; the fake already // relied on checkpoint_dir and restore_dir above to place and find it. for _, rpc := range []string{"RunWorkload", "CheckpointWorkload", "RestoreWorkload", "TerminateWorkload"} { want := ateletpath.ActorDirs(actorUID) @@ -1058,7 +1059,7 @@ func TestRestoreWithoutUsableLocalCopy(t *testing.T) { } // TestTerminateWithoutWorkerPodUID verifies that Terminate with an empty -// WorkerPodUid (used when cleaning up the assigned node for a paused or +// WorkerPodUid (used when cleaning up the assigned node for a SUSPENDED or // crashed actor that no longer has a worker pod) skips dialing worker while // still pruning local checkpoints and removing actor directories on the node. func TestTerminateWithoutWorkerPodUID(t *testing.T) { @@ -1592,7 +1593,7 @@ func TestTerminateWaitsForInFlightRPC(t *testing.T) { } // TestFailedCheckpointKeepsLocalSnapshot checkpoints a resumed actor again, and -// fails before the new snapshot commits: the pause snapshot stays on disk and +// fails before the new snapshot commits: the local snapshot stays on disk and // recorded as the actor's latest local snapshot, so the actor can still be // restored from it. func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { @@ -1669,10 +1670,10 @@ func TestFailedCheckpointKeepsLocalSnapshot(t *testing.T) { } } -// TestPromoteSnapshotRecordsUpload suspends a paused actor: the promote -// records the uploaded snapshot as the latest durable one, with the pause's -// commit time, and keeps the local copy on the node. A retry reports the same -// state. +// TestPromoteSnapshotRecordsUpload promotes a suspended actor's local snapshot: +// the promote records the uploaded snapshot as the latest durable one, with the +// checkpoint's commit time, and keeps the local copy on the node. A retry +// reports the same state. func TestPromoteSnapshotRecordsUpload(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("pause-snap-1", 1) @@ -1851,7 +1852,7 @@ func TestTrackerStoreUploadLocalSnapshot(t *testing.T) { } st := &trackerStore{s: a.s} - // An upload asking for a fidelity other than the pause captured is + // An upload asking for a fidelity other than the checkpoint captured is // rejected before uploading. noFidelity := proto.CloneOf(snap) noFidelity.Fidelity = ateletpb.SnapshotFidelity_SNAPSHOT_FIDELITY_UNSPECIFIED @@ -1876,7 +1877,7 @@ func TestTrackerStoreUploadLocalSnapshot(t *testing.T) { } } -// TestRestoredActorIsNotDetached restores a paused actor from its local copy: +// TestRestoredActorIsNotDetached restores an actor from its local copy: // once the actor runs on the node, atelet refuses to detach it. func TestRestoredActorIsNotDetached(t *testing.T) { a := runPluginActor(t) @@ -1890,9 +1891,10 @@ func TestRestoredActorIsNotDetached(t *testing.T) { } } -// TestFailedRestoreUnschedulesDetach fails a restore of a paused actor after -// it began: as after a restore that succeeds, atelet keeps the actor's local -// snapshot, and does not detach the actor until its next checkpoint. +// TestFailedRestoreUnschedulesDetach fails a restore of an actor from its local +// snapshot after it began: as after a restore that succeeds, atelet keeps the +// actor's local snapshot, and does not detach the actor until its next +// checkpoint. func TestFailedRestoreUnschedulesDetach(t *testing.T) { a := runPluginActor(t) a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) @@ -1923,7 +1925,7 @@ func TestFailedRestoreUnschedulesDetach(t *testing.T) { } } -// TestRestoreAfterDetachReadsUpload resumes a paused actor that atelet +// TestRestoreAfterDetachReadsUpload resumes a suspended actor that atelet // detached before the control plane recorded the upload, so the control plane // still sends the Restore here: atelet restores the actor from the upload the // request names. @@ -1968,9 +1970,9 @@ func detachActorRequest() *ateletpb.DetachActorRequest { return &ateletpb.DetachActorRequest{Atespace: "ate-demo", ActorName: "counter", ActorUid: pluginActorUID} } -// TestDetachActorDetachesPausedActor detaches a paused actor through the RPC: -// atelet uploads its local snapshot, deletes the local copy, and returns the -// new state. +// TestDetachActorDetachesPausedActor detaches a suspended actor through the +// RPC: atelet uploads its local snapshot, deletes the local copy, and returns +// the new state. func TestDetachActorDetachesPausedActor(t *testing.T) { a := runPluginActor(t) a.s.snapshots = snapshottracker.New(t.Context(), &trackerStore{s: a.s}, clock.RealClock{}) @@ -2123,8 +2125,8 @@ func (s *blockingTrackerStore) UploadLocalSnapshot(ctx context.Context, _ string return false, ctx.Err() } -// startBlockedDetach pauses the actor as pause-snap-1, and starts a Detach of -// it whose upload blocks until the test ends. +// startBlockedDetach checkpoints the actor locally as pause-snap-1, and starts +// a Detach of it whose upload blocks until the test ends. func startBlockedDetach(t *testing.T, a *pluginActor) { t.Helper() store := &blockingTrackerStore{started: make(chan struct{}, 1)} @@ -2206,10 +2208,10 @@ func TestPromoteSnapshotRecordsDetachWait(t *testing.T) { } } -// TestPromoteSnapshotFoundAlreadyUploaded promotes a paused actor's snapshot -// whose upload committed before the promote, unrecorded, and whose local copy -// is gone since: the promote records the snapshot as durable, without an -// upload time, since it wrote nothing. +// TestPromoteSnapshotFoundAlreadyUploaded promotes an asynchronously suspended +// actor's snapshot whose upload committed before the promote, unrecorded, and +// whose local copy is gone since: the promote records the snapshot as durable, +// without an upload time, since it wrote nothing. func TestPromoteSnapshotFoundAlreadyUploaded(t *testing.T) { a := runPluginActor(t) snap := a.snapshot("pause-snap-1", 1) @@ -2458,8 +2460,8 @@ func TestPromoteSnapshotWaitsForBackgroundUpload(t *testing.T) { } } -// TestCheckpointKeepsSnapshotBeingUploaded pauses a resumed actor while the -// background upload of its previous pause snapshot still runs: the +// TestCheckpointKeepsSnapshotBeingUploaded checkpoints a resumed actor locally +// while the background upload of its previous local snapshot still runs: the // checkpoint's prune keeps that snapshot for the upload, which deletes it once // it ends. func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { @@ -2509,7 +2511,7 @@ func TestCheckpointKeepsSnapshotBeingUploaded(t *testing.T) { } // TestImmediateCheckpointCancelsBackgroundUpload suspends a resumed actor with -// an immediate upload while the background upload of its previous pause +// an immediate upload while the background upload of its previous local // snapshot still runs: the checkpoint checkpoints the worker without waiting // for that upload, then cancels it, and returns only once it has stopped, so // the control plane can delete the older snapshot's objects. @@ -2594,7 +2596,7 @@ func TestImmediateCheckpointCancelsBackgroundUpload(t *testing.T) { } } -// resumedWithPendingUpload runs a plugin actor, pauses it into pause-snap-1 +// resumedWithPendingUpload runs a plugin actor, checkpoints it into pause-snap-1 // with a deferred upload, and resumes it from that snapshot's local copy // before the upload is due. It returns the actor, its gated tracker store, and // a func that steps the clock past the upload's due time and reports whether @@ -2646,7 +2648,7 @@ func resumedWithPendingUpload(t *testing.T) (*pluginActor, *gatingTrackerStore, } // TestImmediateCheckpointHoldsOffBackgroundUpload checkpoints an actor resumed -// from its pause snapshot while that snapshot's background upload comes due: +// from its local snapshot while that snapshot's background upload comes due: // under UPLOAD_MODE_IMMEDIATE the upload does not start, since the checkpoint // would only cancel it, while under UPLOAD_MODE_DEFERRED it starts. func TestImmediateCheckpointHoldsOffBackgroundUpload(t *testing.T) { @@ -2674,7 +2676,7 @@ func TestImmediateCheckpointHoldsOffBackgroundUpload(t *testing.T) { } // TestTerminateHoldsOffBackgroundUpload terminates an actor resumed from its -// pause snapshot, whose background upload comes due while the worker tears the +// local snapshot, whose background upload comes due while the worker tears the // actor down: the upload does not start, since Terminate would only cancel it. // When the worker fails to tear the actor down, Terminate fails before it // forgets the actor, and the upload, still due, starts once Terminate returns. diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 42d5029be1..5f66c54792 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -1158,8 +1158,9 @@ func (s *Atelet) cleanUpAfterCheckpoint(ctx context.Context, req *ateletpb.Check // Do not move this above CheckpointWorkload to keep MergeDeltaIntoBase on its // in-place path: that leaves the whole checkpoint window with no local snapshot // while LocalSnapshotInfo still names the pruned one, and a crash there strands - // the actor for good (a paused actor has no uploaded copy for resume to fall - // back to, RequiredNodes pins it to this node, nothing clears the field). + // the actor for good (before the background upload completes, the actor has no + // uploaded copy for resume to fall back to, RequiredNodes pins it to this node, + // and nothing clears the field). // // Best-effort: if this fails, the actor's next checkpoint or terminate // prunes again. @@ -1328,11 +1329,11 @@ func (s *Atelet) uploadSnapshot(ctx context.Context, actor actorRef, uri resourc return nil } -// PromoteSnapshot uploads a paused actor's local checkpoint to object -// storage so the actor can be suspended and is no longer tied to this node. It -// drives no worker — the actor's sandbox is gone; the checkpoint files and their -// self-describing manifest already sit under the actor's local-checkpoints -// directory, written by an earlier local Checkpoint (pause). +// PromoteSnapshot uploads a suspended actor's local checkpoint to object +// storage while keeping the local copy on this node (e.g. for CreateTag). It +// drives no worker — the actor's sandbox is gone; the checkpoint files and +// their self-describing manifest already sit under the actor's +// local-checkpoints directory, written by an earlier Checkpoint. func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnapshotRequest) (_ *ateletpb.PromoteSnapshotResponse, err error) { if errs := apivalidation.ValidatePromoteSnapshotRequest(ctx, req); len(errs) > 0 { return nil, resources.ToAPIError(errs) @@ -1343,7 +1344,7 @@ func (s *Atelet) PromoteSnapshot(ctx context.Context, req *ateletpb.PromoteSnaps op := snapshotOp{ templateNamespace: req.GetActorTemplateAtespace(), templateName: req.GetActorTemplateName(), - // Uploading a paused checkpoint creates a durable latest snapshot. + // Uploading a local checkpoint creates a durable latest snapshot. kind: ateattr.SnapshotKindLatest, fidelity: ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()), } @@ -1440,14 +1441,13 @@ func (s *Atelet) uploadLocalCheckpointDir(ctx context.Context, req *ateletpb.Pro capturedFidelity := rec.Fidelity if capturedFidelity == "" { - return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and pause the actor again before suspending it", localSnapshotName) + return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot %q has no fidelity recorded in its manifest; resume and suspend the actor again", localSnapshotName) } - // Pause and suspend capture the template's one preferred fidelity, and a - // paused actor's template cannot change, so the upload is always of what - // the pause captured. A mismatch means the store or the control plane - // drifted; refuse rather than guess. + // Every suspend records the fidelity it captured on the snapshot, so the + // upload is always of what the checkpoint captured. A mismatch means the + // store or the control plane drifted; refuse rather than guess. if desiredFidelity := ateattr.SnapshotFidelityValue(req.GetSnapshot().GetFidelity()); capturedFidelity != desiredFidelity { - return rec.SandboxClass, false, apierror.FailedPrecondition("pause snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) + return rec.SandboxClass, false, apierror.FailedPrecondition("local snapshot captured %s; upload asks for %s", capturedFidelity, desiredFidelity) } // The local snapshot stays until the upload succeeds, so a retry can @@ -1866,7 +1866,7 @@ func (s *Atelet) Terminate(ctx context.Context, req *ateletpb.TerminateRequest) return nil, fmt.Errorf("failed to unmount external volumes during terminate (actor: %s, actorUID: %s): %w", actorRef, actorUID, err) } - // The actor is gone, so no pause snapshot of it can ever be restored again. + // The actor is gone, so no local snapshot of it can ever be restored again. // Forget cancels the actor's background upload or detach in progress and // waits for it to end before this prunes the actor's files. // TODO(#664): this only removes local snapshots in one node. We should clean diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 35018affb6..dcadeb47d5 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -1272,7 +1272,7 @@ func (r *recordingObjectStorage) keys() []string { return keys } -// writeLocalSnapshot lays a local pause snapshot out in dir: the given files +// writeLocalSnapshot lays a node-local snapshot out in dir: the given files // plus the marshaled manifest beside them. func writeLocalSnapshot(t *testing.T, dir string, rec sandboxAssetsRecord, contents map[string]string) { t.Helper() diff --git a/cmd/atelet/metrics.go b/cmd/atelet/metrics.go index 84f8ff835d..0fb01a77e2 100644 --- a/cmd/atelet/metrics.go +++ b/cmd/atelet/metrics.go @@ -166,9 +166,9 @@ func restoreSnapshotKind(directLocal bool, rec *sandboxAssetsRecord) string { return ateattr.SnapshotKindLatest } -// checkpointSnapshotKind classifies which snapshot a checkpoint writes: an -// asynchronous suspend writes the node-local one, and a synchronous suspend -// writes a durable latest snapshot. +// checkpointSnapshotKind classifies which snapshot a checkpoint writes: a +// deferred-upload checkpoint (async suspend) writes the node-local one, and an +// immediate-upload checkpoint (sync suspend) writes a durable latest snapshot. func checkpointSnapshotKind(req *ateletpb.CheckpointRequest) string { if req.GetUploadConfig().GetMode() == ateletpb.UploadMode_UPLOAD_MODE_DEFERRED { return ateattr.SnapshotKindLocal diff --git a/cmd/atelet/snapshot_transfer_test.go b/cmd/atelet/snapshot_transfer_test.go index 466ac3c850..d46518c376 100644 --- a/cmd/atelet/snapshot_transfer_test.go +++ b/cmd/atelet/snapshot_transfer_test.go @@ -437,7 +437,7 @@ func (s *pluginSidecar) stop() { } // The upload after CheckpointWorkload waits out a plugin outage longer than -// the ready wait, since a retried Checkpoint cannot redo it. The paused +// the ready wait, since a retried Checkpoint cannot redo it. The PromoteSnapshot // upload, which a retry can redo, still fails within the ready wait. func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { const wait = 200 * time.Millisecond @@ -512,7 +512,7 @@ func TestCheckpointUploadWaitsOutPluginOutage(t *testing.T) { } } -// fetchManifest keeps the plugin's NotFound readable, which the paused +// fetchManifest keeps the plugin's NotFound readable, which the local // snapshot upload uses to tell a missing snapshot from a failed probe. func TestFetchManifestKeepsNotFound(t *testing.T) { s := &Atelet{ diff --git a/cmd/atelet/snapshotreport_test.go b/cmd/atelet/snapshotreport_test.go index b74c290a92..4285864dc6 100644 --- a/cmd/atelet/snapshotreport_test.go +++ b/cmd/atelet/snapshotreport_test.go @@ -170,9 +170,10 @@ func nextReport(t *testing.T, clk *clocktesting.FakeClock, workers *fakeReportSe return got } -// TestBackgroundUploadAndDetachAreReported pauses an actor and leaves it -// paused: atelet reports the background upload of its snapshot, then its -// background detach, to the control plane, for the actor its Checkpoint named. +// TestBackgroundUploadAndDetachAreReported checkpoints an actor locally and +// leaves it suspended: atelet reports the background upload of its snapshot, +// then its background detach, to the control plane, for the actor its +// Checkpoint named. func TestBackgroundUploadAndDetachAreReported(t *testing.T) { a := runPluginActor(t) clk := clocktesting.NewFakeClock(time.Now()) diff --git a/cmd/atelet/workersupport.go b/cmd/atelet/workersupport.go index 3b034e16b3..85cdc2e724 100644 --- a/cmd/atelet/workersupport.go +++ b/cmd/atelet/workersupport.go @@ -207,8 +207,8 @@ func toAteAPICompat(in *ateletpb.VersionedSandboxCompat) *ateapipb.VersionedSand // // The control plane's error is returned unwrapped: a refusal is a real answer // here -- the actor is no longer assigned to this worker, or the suspend lost a -// race to a resume, pause, or delete -- and the worker needs to tell those from -// a transport failure it should retry. +// race to a resume, suspend, or delete -- and the worker needs to tell those +// from a transport failure it should retry. func (s *workerSupportServer) RequestActorSuspend(ctx context.Context, req *ateletpb.RequestActorSuspendRequest) (*ateletpb.RequestActorSuspendResponse, error) { // Identity comes only from the mTLS certificate, never from the request: a // worker can speak for the actors it hosts and no others. Which those are diff --git a/docs/api-guide.md b/docs/api-guide.md index 09840280e9..97050445cd 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -100,7 +100,7 @@ The `ActorTemplate` defines the code, environment, and state-management policies | :--- | :--- | :--- | | `containers` | `[]Container` | **Required.** The workload definition — see [Container Fields](#container-fields) below. Each container may also declare an optional `wakeupProbe` HTTP probe — see [Container Wakeup Probe](#container-wakeup-probe-wakeupprobe). | | `workerSelector` | `*Selector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels (`matchLabels`). If unset, all pools are eligible (subject to the actor's own `worker_selector`). The pool decides the sandbox class, so this is also how a template pins one: if your cluster has pools of more than one sandbox class and the workload needs a particular class, make sure the selector points only at pools of that class. An actor booting with no snapshot and no selector may land on any class, and keeps it from its first checkpoint on. | -| `snapshotConfig` | `SnapshotConfig` | Optional; omitted fields take their defaults. The base object-storage location snapshots are written under (a preview field, ignored without the `PerTemplateSnapshotLocation` gate), the snapshot fidelity (`preferredFidelity`) every Pause and Suspend captures, and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | +| `snapshotConfig` | `SnapshotConfig` | Optional; omitted fields take their defaults. The base object-storage location snapshots are written under (a preview field, ignored without the `PerTemplateSnapshotLocation` gate), the snapshot fidelity (`preferredFidelity`) every Suspend captures, and automated golden snapshot creation (`goldenSnapshotConfig.mode`). See [Snapshot Storage Layout](#snapshot-storage-layout). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each a `durableDir`, an `external` CSI volume source (see [CSI Volumes Guide](csi-volumes.md)), or a `systemInfo` volume (see [SystemInfo Volumes](#systeminfo-volumes)). Every declared volume must be mounted by at least one container. A `durableDir` is created `0777`, as Kubernetes creates an emptyDir, so a container running as any user can write it. A snapshot restores it with the mode it had, so the container's own chmod survives suspend and resume. | | `resources` | `*ResourceRequirements` | Required. Declares each actor's compute size via `limits`, which must set both `cpuMillicores` and `memoryBytes` — see [Sandbox Right-Sizing](#sandbox-right-sizing-resources). Immutable, like the rest of the template. | @@ -379,9 +379,11 @@ a `Snapshot` with the following fields: see `ActorStatus.assigned_node` in the [API schema](../pkg/proto/ateapipb/ateapi.proto). - `durableSnapshot`: The copy in object storage, with its URI in `object.snapshotUri` and a `status` that is `PENDING` while the copy is being - written and `COMPLETED` once it can be restored from. A pause records the copy - as `PENDING`, and atelet uploads it in the background. A tag's snapshot always - has one, and the tag can seed actors only once it is `COMPLETED`. + written and `COMPLETED` once it can be restored from. A synchronous suspend + uploads the copy before returning (`COMPLETED`); an asynchronous suspend + records the copy as `PENDING`, and atelet uploads it in the background. A + tag's snapshot always has one, and the tag can seed actors only once it is + `COMPLETED`. - `sandboxRuntime`: The sandbox runtime and hardware compatibility attributes the guest state was captured under, copied from `status.workerAssignment.sandboxRuntime` when the checkpoint is taken. A resume that restores memory state @@ -521,7 +523,7 @@ Golden snapshot creation is enabled by default. To disable it, set template. Omitting the config or leaving `mode` unspecified defaults to `GOLDEN_SNAPSHOT_MODE_ENABLED`. A disabled template creates no golden actor or tag. Actors created without a `sourceTag` cold-boot and can still -pause, suspend, and resume using their own snapshots. +suspend and resume using their own snapshots. `kubectl ate get actor-templates` shows `DISABLED` in the `GOLDEN TAG` column for these templates. @@ -605,14 +607,14 @@ Replaces the mutable fields of an existing actor with the ones in the request (` * `actor`: `Actor` — the complete replacement actor. `metadata.atespace` and `metadata.name` identify the resource; `metadata.uid` and `metadata.version` are **required** preconditions. `metadata` and `status` are server-owned and whatever the request carries in them is ignored. `source_tag` is immutable. * `actor_template` may change only while the actor is `SUSPENDED`. The new template must declare the same volumes and volume mounts. Once the actor owns an external snapshot, the new template must also store snapshots under the same location: its `snapshotConfig.storageLocation` with the `PerTemplateSnapshotLocation` preview gate, otherwise ate-api-server's `--snapshot-location` (see [Snapshot Storage Layout](#snapshot-storage-layout)). * `worker_selector` may be updated or cleared (`nil`) in any state; the updated selector takes effect on the next `ResumeActor`. - * A paused actor is initially pinned to its node (see `locality` in [Snapshot Storage Layout](#snapshot-storage-layout)). `ResumeActor` first looks for a worker there matching the template and actor selectors. If none is available but a worker satisfies the remaining constraints without node or runtime restrictions, resume detaches the checkpoint and retries placement. If no eligible worker has room, it returns `RESOURCE_EXHAUSTED`. + * A suspended actor with a local snapshot is initially pinned to its node (see `locality` in [Snapshot Storage Layout](#snapshot-storage-layout)). `ResumeActor` first looks for a worker there matching the template and actor selectors. If none is available but a worker satisfies the remaining constraints without node or runtime restrictions, resume detaches the checkpoint and retries placement. If no eligible worker has room, it returns `RESOURCE_EXHAUSTED`. * **Response:** the updated `Actor`. * **Errors:** `INVALID_ARGUMENT` if `uid` or `version` is unset, or if the request changes an immutable field — including by leaving one unset; `FAILED_PRECONDITION` if a change to `actor_template` breaks one of the rules above; `ABORTED` if either guard no longer matches the stored resource. Because the guards are required and only a read supplies them, an update is always a read-modify-write. To Update an `Actor`, you must first `GetActor`/`CreateActor`, instead of building a new one — see [§7.2 of the API style guide](api-style-guide.md#72-using-version-and-uid-to-guard-writes) for why reconstructing the message can silently drop data. #### `ResumeActor` -Activates a suspended or paused actor by restoring it onto a physical worker. +Activates a suspended actor by restoring it onto a physical worker. * **Request:** `ResumeActorRequest` * `actor`: `ObjectRef` of the actor to resume. * **Response:** `ResumeActorResponse` containing the updated `Actor` object (including the physical worker placement in `status.worker_assignment`). @@ -621,9 +623,9 @@ Activates a suspended or paused actor by restoring it onto a physical worker. Hibernate a running actor, capturing its current RAM and disk state into a snapshot. * **Request:** `SuspendActorRequest` * `actor`: `ObjectRef` of the actor to suspend. - * `options.mode`: (Optional) `DURABILITY_MODE_SYNCHRONOUS` returns once the snapshot is `COMPLETED` in object storage. `DURABILITY_MODE_ASYNCHRONOUS` (a pause) returns once the snapshot is captured on the worker's node, and atelet uploads it in the background. Unset runs an asynchronous suspend. + * `options.mode`: (Optional) `DURABILITY_MODE_SYNCHRONOUS` returns once the snapshot is `COMPLETED` in object storage. `DURABILITY_MODE_ASYNCHRONOUS` returns once the snapshot is captured on the worker's node, and atelet uploads it in the background. Unset runs an asynchronous suspend. * **Response:** `SuspendActorResponse` containing the `Actor` object in `ACTOR_STATE_SUSPENDED`, with its snapshot in `status.snapshots`. -* Called on an actor that is already `ACTOR_STATE_SUSPENDED`, it returns the actor as it is, in either mode. atelet uploads a paused actor's node-local snapshot in the background, so `status.snapshots` may still show it with `locality` and a `PENDING` `durableSnapshot`. +* Called on an actor that is already `ACTOR_STATE_SUSPENDED`, it returns the actor as it is, in either mode. After an asynchronous suspend, atelet uploads the snapshot in the background, so `status.snapshots` may still show a `PENDING` `durableSnapshot`. * A snapshot generation conflict returns `ABORTED` without crashing the actor. The actor can remain `ACTOR_STATE_SUSPENDING`, with its worker assignment and previous completed snapshot kept for recovery. Read the actor's latest state before retrying; repeating a stale request cannot resolve a persistent disagreement with atelet. * A successful suspend releases the actor's previous external snapshot: an actor keeps one, and only tags outlive it. To keep the snapshot a suspend just wrote, tag it with `CreateTag` while the actor is still suspended. After an asynchronous suspend, `CreateTag` first has atelet upload the snapshot if it has not already, so the call can take as long as that upload. If the upload fails, so does `CreateTag`, and the actor is left as it was. @@ -645,9 +647,9 @@ Deletion always runs before the database reference is dropped, and a failure fai #### `RevertActor` Discards an actor's live or crashed execution and transitions it to `ACTOR_STATE_SUSPENDED` at its last completed external snapshot (`status.snapshots`). * **Request:** `RevertActorRequest` - * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING` and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries), and from a paused actor: `ACTOR_STATE_SUSPENDED` whose latest snapshot is node-local. Calling `RevertActor` on an `ACTOR_STATE_SUSPENDED` actor with no node-local snapshot returns `FAILED_PRECONDITION`. + * `actor`: `ObjectRef` of the actor to revert. Accepted from `ACTOR_STATE_RUNNING` and `ACTOR_STATE_CRASHED` (plus `ACTOR_STATE_REVERTING` for idempotent retries), and from `ACTOR_STATE_SUSPENDED` when the actor's latest snapshot is node-local. Calling `RevertActor` on an `ACTOR_STATE_SUSPENDED` actor with no node-local snapshot returns `FAILED_PRECONDITION`. * **Response:** `RevertActorResponse` containing the reverted `Actor` in `ACTOR_STATE_SUSPENDED`. -* Reverting terminates any bound worker sandbox, clears node-local pause checkpoints, and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed durable snapshot in `status.snapshots`. +* Reverting terminates any bound worker sandbox, clears node-local checkpoints, and garbage-collects any partial external snapshot left by an interrupted suspend while preserving the last committed durable snapshot in `status.snapshots`. * External volumes are not reverted. Their contents are never part of a snapshot, so a reverted actor comes back with its memory and root filesystem rewound but its volumes exactly as the discarded execution left them. #### `DeleteActor` diff --git a/docs/api-style-guide.md b/docs/api-style-guide.md index 484576c71c..c599444f54 100644 --- a/docs/api-style-guide.md +++ b/docs/api-style-guide.md @@ -10,7 +10,7 @@ This guide is derived from [Google's AIP (API Improvement Proposals)](https://go *Follows [AIP-121](https://google.aip.dev/121).* -APIs are structured around **resources** (nouns) and a small set of standard **methods** (verbs). Standard methods are Get, List, Create, Update, and Delete. **Custom methods** handle operations that don't fit these patterns (e.g., Suspend, Resume, Pause for Actors). +APIs are structured around **resources** (nouns) and a small set of standard **methods** (verbs). Standard methods are Get, List, Create, Update, and Delete. **Custom methods** handle operations that don't fit these patterns (e.g., Suspend, Resume, Revert for Actors). Rules: - Every primary noun the API exposes is a resource. @@ -333,7 +333,7 @@ allow external controllers and upstream systems / automations to react to resour *Follows [AIP-136](https://google.aip.dev/136).* -Custom methods are for operations that don't map cleanly to CRUD: lifecycle transitions (Suspend, Resume, Pause), long-running actions, or commands with side effects that standard Update semantics would misrepresent. +Custom methods are for operations that don't map cleanly to CRUD: lifecycle transitions (Suspend, Resume, Revert), long-running actions, or commands with side effects that standard Update semantics would misrepresent. ```proto rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} diff --git a/docs/architecture.md b/docs/architecture.md index fa8a07ffb3..708aaea427 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -423,27 +423,28 @@ stateDiagram-v2 [*] --> SUSPENDED : CreateActor SUSPENDED --> RESUMING : ResumeActor RESUMING --> RUNNING : restore / boot complete - RUNNING --> SUSPENDING : SuspendActor (sync) + RUNNING --> SUSPENDING : SuspendActor SUSPENDING --> SUSPENDED : checkpoint complete - RUNNING --> SUSPENDING : SuspendActor (async, the default: keeps the snapshot on the node) SUSPENDED --> [*] : DeleteActor ``` -A pause and a suspend both pass through `SUSPENDING`, and a paused actor and -a suspended actor are both `SUSPENDED`. They differ only in the shape of -`status.snapshots`: - - * After a pause (an asynchronous suspend), the latest snapshot has a - `locality` (the node holding its files) and a `PENDING` - `durableSnapshot`. `status.assignedNode` pins the actor to that node, so - `ResumeActor` restores it there from the local files. - * atelet uploads a paused actor's snapshot in the background - (`--snapshot-upload-delay`) and reports it, which marks the - `durableSnapshot` `COMPLETED`. `--snapshot-detach-delay` after the pause, - atelet deletes the local files and reports that too, which clears the - `locality` and `status.assignedNode`. - * After a synchronous suspend, the latest snapshot has a `COMPLETED` - `durableSnapshot` and the actor is pinned to no node. +Both synchronous and asynchronous `SuspendActor` pass through `SUSPENDING`, +leave the actor in `SUSPENDED`, and keep a local copy of the snapshot on the +node (`locality`). They differ in when the snapshot is uploaded to object +storage: + + * After an asynchronous suspend (the default), `SuspendActor` returns as soon + as the local checkpoint is written, with a `PENDING` `durableSnapshot`. + `atelet` uploads the snapshot in the background (`--snapshot-upload-delay`) + and reports it, which marks the `durableSnapshot` `COMPLETED`. + * After a synchronous suspend, `atelet` uploads the snapshot before + `SuspendActor` returns, so the `durableSnapshot` is immediately + `COMPLETED`. + * While the latest snapshot has `locality`, `ResumeActor` restores from the + local files on that node. `--snapshot-detach-delay` after the suspend (or + during `ResumeActor` when that node has no room), `atelet` deletes the + local files (uploading the snapshot first if needed), clearing `locality` + and `status.assignedNode`. `SuspendActor` on a `SUSPENDED` actor returns it as it is, in either mode. @@ -485,10 +486,11 @@ Triggered by an explicit `SuspendActor` call. 1. **Checkpoint**: The `atelet` instructs `ateworker` to freeze the process and capture a memory+disk snapshot. - 2. **Persistence**: The `atelet` uploads the snapshot to durable storage - (e.g., GCS). With `DURABILITY_MODE_SYNCHRONOUS` the upload finishes before - `SuspendActor` returns. With the asynchronous default the snapshot stays - on the node and `atelet` uploads it in the background. + 2. **Persistence**: The `atelet` keeps a local copy of the snapshot on the + node and uploads it to durable storage (e.g., GCS). With + `DURABILITY_MODE_SYNCHRONOUS` the upload finishes before `SuspendActor` + returns. With the asynchronous default `SuspendActor` returns once the + local checkpoint is written, and `atelet` uploads it in the background. 3. **Reclaim**: The physical worker is wiped and returned to the `WorkerPool`. @@ -510,7 +512,7 @@ tags cannot be deleted until they are. ### Phase 4: Recovery (`RevertActor`) and Deletion (`DeleteActor`) -An actor in `ACTOR_STATE_RUNNING` or `ACTOR_STATE_CRASHED`, or a paused actor (`ACTOR_STATE_SUSPENDED` with a node-local latest snapshot), can be reverted back to `ACTOR_STATE_SUSPENDED` at its durable snapshot via `RevertActor`. Reverting terminates any active sandbox, discards any local pause checkpoint or partial in-progress snapshot, and preserves the actor's last completed external snapshot so a subsequent `ResumeActor` restores from that checkpoint. +An actor in `ACTOR_STATE_RUNNING` or `ACTOR_STATE_CRASHED`, or an `ACTOR_STATE_SUSPENDED` actor with a node-local latest snapshot, can be reverted back to `ACTOR_STATE_SUSPENDED` at its durable snapshot via `RevertActor`. Reverting terminates any active sandbox, discards any node-local checkpoint or partial in-progress snapshot, and preserves the actor's last completed external snapshot so a subsequent `ResumeActor` restores from that checkpoint. An actor in any state (such as `ACTOR_STATE_RUNNING`) can be deleted from the Control Plane. The workflow terminates the running containers on the worker, detaches mounted volumes, and frees the worker assignment before deleting the record. diff --git a/docs/authorization.md b/docs/authorization.md index 2e6f1c79c7..6745a9d7d1 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -117,7 +117,7 @@ one **atespace**. | Create, update, delete access policies | ✅ | | ✅ (its own) | | | | Get and list actors, actor templates, tags, egress policies | ✅ | ✅ | ✅ | ✅ | ✅ | | Create, update, delete actors and actor templates | ✅ | | ✅ | ✅ | | -| Pause, resume, suspend, revert actors | ✅ | | ✅ | ✅ | | +| Resume, suspend, revert actors | ✅ | | ✅ | ✅ | | | Create, update, delete egress policies | ✅ | | ✅ | ✅ | | | Create, publish, delete tags | ✅ | | ✅ | ✅ | | | Create an actor from a tag (`sourceTag`) | ✅ | | ✅ | ✅ | | diff --git a/docs/glossary.md b/docs/glossary.md index 156bf42d7d..4fe6787cb5 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -83,18 +83,15 @@ for etcd. ## Lifecycle - **Suspend**: hibernate a running Actor into a snapshot. The Actor is - checkpointed on its Worker, which is then freed. A synchronous suspend - uploads the snapshot to external storage before it returns; an asynchronous - one is a Pause. Suspending an Actor that is already suspended or paused - returns it as it is. - -- **Pause**: a short-term checkpoint of a running Actor, taken by an - asynchronous suspend. Snapshot files remain on the node VM, and the following - Resume is prioritized onto the node VM where the snapshots are persisted. - atelet uploads the snapshot to external storage in the background, and ends - the node pinning once it deletes the local files. - -- **Resume**: activate a suspended/paused Actor by restoring it onto a Worker. The + checkpointed on its Worker, which is then freed. Snapshot files remain on the + node VM so the following Resume can restore from them locally (until `atelet` + or a Resume detaches them), and are also uploaded to external storage: a + synchronous suspend uploads the snapshot before it returns, while an + asynchronous suspend (the default) returns once the local snapshot is captured + and `atelet` uploads it in the background. Suspending an Actor that is already + suspended returns it as it is. + +- **Resume**: activate a suspended Actor by restoring it onto a Worker. The common path restores from a snapshot rather than cold-booting. ## Volumes @@ -125,8 +122,8 @@ for etcd. Fidelity describes only what a snapshot *holds*. A template sets one level, `preferredFidelity`, the ceiling for every snapshot of its actors: - the node-local checkpoint a [Pause](#lifecycle) keeps on the node and the - snapshot a [Suspend](#lifecycle) uploads to snapshot storage. It is best + both the node-local checkpoint a [Suspend](#lifecycle) keeps on the node and + the snapshot uploaded to snapshot storage. It is best effort: the system never packages more than that level, and may package less when a layer cannot be captured or is too expensive to capture at the time. A process resumed in place keeps its memory regardless. diff --git a/docs/metrics/registry/metrics.yaml b/docs/metrics/registry/metrics.yaml index a3caa11f0d..cd50ce92ab 100644 --- a/docs/metrics/registry/metrics.yaml +++ b/docs/metrics/registry/metrics.yaml @@ -104,8 +104,9 @@ groups: stability: development value: suspending brief: > - ateapi is writing a snapshot: node-local for an asynchronous - suspend, durable for a synchronous suspend. + ateapi is writing a snapshot: uploaded to object storage before + returning for a synchronous suspend, or in the background for + an asynchronous suspend. - id: suspended stability: development value: suspended @@ -299,9 +300,11 @@ groups: stability: development value: local brief: > - The copy on the node. An asynchronous suspend writes it. A - resume reads it while the node still has it, after either kind - of suspend and whether or not the upload has finished. + The copy on the node. On checkpoint, an asynchronous suspend + writes local (whereas a synchronous suspend writes latest, + though it also keeps a copy on the node). On restore, a resume + reads local while the node still has the copy, after either + kind of suspend and whether or not the upload has finished. - id: latest stability: development value: latest diff --git a/internal/ateworker/suspend/suspend.go b/internal/ateworker/suspend/suspend.go index 08ac36f2c6..0757fd6f9f 100644 --- a/internal/ateworker/suspend/suspend.go +++ b/internal/ateworker/suspend/suspend.go @@ -112,7 +112,7 @@ type Actor struct { // The caller's own context bounds the whole thing. // // A refusal is not retried. The control plane declining -- the actor is no -// longer assigned to this worker, or a resume, pause, or delete won the race -- +// longer assigned to this worker, or a resume, suspend, or delete won the race -- // is an answer, and asking again cannot change it. func (r *Requester) RequestSuspend(ctx context.Context, actor Actor) error { // A fresh connection per request picks up rotated worker credentials and @@ -166,7 +166,7 @@ func requestWithRetry(ctx context.Context, conn grpc.ClientConnInterface, actor // holds the actor's lease, or a write lost to a concurrent one. The lease is // the likely one here, because a suspend takes it before it does anything // else, and it is worth waiting out -- whatever holds it finishes, and the -// retry re-runs every precondition against fresh state, so a pause or delete +// retry re-runs every precondition against fresh state, so a suspend or delete // that won the race is refused rather than overridden. // // Everything else is left alone. NotFound and FailedPrecondition are diff --git a/internal/benchmarking/boomer/agentsession/agentsession.go b/internal/benchmarking/boomer/agentsession/agentsession.go index c1bbbbd6c4..d539d39381 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession.go +++ b/internal/benchmarking/boomer/agentsession/agentsession.go @@ -633,9 +633,9 @@ type sessionUser struct { // ingestBuf is the runtime's shared read-only ingest payload. ingestBuf []byte // awake is true when the actor is resident (RUNNING) rather than - // suspended or paused. Zero value (false) means parked. + // suspended. Zero value (false) means parked. awake bool - // hibernatePending is set by a failed Pause/Suspend: the actor is + // hibernatePending is set by a failed SuspendActor: the actor is // stranded RUNNING or SUSPENDING. Waking it from there would misreport // WakeFirstTouch, so runStep finishes the hibernate first. hibernatePending bool diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig.go b/internal/benchmarking/boomer/dynconfig/dynconfig.go index e91633d01b..a107ced68d 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig.go @@ -240,8 +240,9 @@ func Uniform(lo, hi time.Duration) time.Duration { // Resume and lifecycle modes. Explicit issues a ResumeActor RPC before // sending traffic; implicit issues no wake request at all: the actor stays // suspended until a request reaches the atenet router, which wakes it -// while the request is parked. Suspend writes the snapshot to durable -// storage; pause keeps it on the node. +// while the request is parked. Suspend runs a synchronous SuspendActor +// (uploads the snapshot before returning); pause runs an asynchronous +// SuspendActor (uploads the snapshot in the background). const ( ResumeModeExplicit = "explicit" ResumeModeImplicit = "implicit" diff --git a/internal/benchmarking/boomer/glutton/lifecycle.go b/internal/benchmarking/boomer/glutton/lifecycle.go index af34d78c6e..985ef5e7f1 100644 --- a/internal/benchmarking/boomer/glutton/lifecycle.go +++ b/internal/benchmarking/boomer/glutton/lifecycle.go @@ -340,7 +340,7 @@ type gluttonActor struct { // ACTOR_STATE_CRASHED (codes.Aborted with "crashed" in the message). // ateapi never rehabilitates one, so iterate() replaces it. crashed bool - // hibernatePending is set by a failed Pause/Suspend: the actor is + // hibernatePending is set by a failed SuspendActor: the actor is // stranded RUNNING or SUSPENDING, which resume has no edge out of. hibernatePending bool // deleted is set once replaceActor has issued this actor's DeleteActor, diff --git a/internal/benchmarking/boomer/resumepaused/resumepaused.go b/internal/benchmarking/boomer/resumepaused/resumepaused.go index 4d737c0f0a..c317bcd89d 100644 --- a/internal/benchmarking/boomer/resumepaused/resumepaused.go +++ b/internal/benchmarking/boomer/resumepaused/resumepaused.go @@ -13,13 +13,13 @@ // limitations under the License. // Package resumepaused drives ate-api-server's cold resume path: every -// ResumeActor targets a paused actor, so ateapi takes the actor lease, binds -// a worker, and waits on an atelet restore. Each resume is followed by a -// pause that makes the actor cold again. Run on the fake data plane +// ResumeActor targets a suspended actor, so ateapi takes the actor lease, +// binds a worker, and waits on an atelet restore. Each resume is followed by a +// suspend that makes the actor cold again. Run on the fake data plane // (benchmarking/workloads/deploy.sh --fake-data-plane), it measures what a // cold resume costs ateapi and Postgres without the router or the restore in -// the way. The pauses run at the resume rate, so a capacity it finds is one -// of resume and pause cycles, not of resumes alone. It sends the actors +// the way. The suspends run at the resume rate, so a capacity it finds is one +// of resume and suspend cycles, not of resumes alone. It sends the actors // themselves no requests, since none exist there. package resumepaused diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 26d7818934..1bbe4ea075 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -276,8 +276,8 @@ func TestDurableDirLifecycle(t *testing.T) { }, }, { - // Pause captures the same VOLUMES fidelity as suspend, so memory - // and rootfs writes cold-boot on every resume. + // Async suspend captures the same VOLUMES fidelity as sync suspend, + // so memory and rootfs writes cold-boot on every resume. name: "preferredFidelity:VOLUMES", tc: actorLifecycleTestCase{ fidelity: ateapipb.SnapshotFidelity_SNAPSHOT_FIDELITY_VOLUMES, @@ -314,7 +314,7 @@ func TestDurableDirLifecycle(t *testing.T) { } // TestMultipleDurableDirLifecycle covers an Actor with TWO durable-dir volumes: -// both must survive pause/resume and suspend/resume independently. +// both must survive async suspend/resume and sync suspend/resume independently. func TestMultipleDurableDirLifecycle(t *testing.T) { t.Parallel() tests := []struct { @@ -547,13 +547,13 @@ func TestExternalVolume_NodeMigration(t *testing.T) { } } -// Verify that file and memory counters behavior after pause and suspend, for different snapshot scopes. +// Verify that file and memory counters behavior after async and sync suspend, for different snapshot scopes. // Test case: // 1. Create actor. // 2. Call to actor and validate memory and file counters. -// 3. Pause & Resume actor. +// 3. Async Suspend & Resume actor. // 4. Call to actor and validate memory and file counters. -// 5. Suspend & Resume actor. +// 5. Sync Suspend & Resume actor. // 6. Call to actor and validate memory and file counters. type actorLifecycleTestCase struct { fidelity ateapipb.SnapshotFidelity @@ -1169,9 +1169,10 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } validateCounterResponse(t, resp, "after revert from RUNNING", 2, 2) - // Revert from paused. Resume prefers a local snapshot over the external - // one, so clearing LocalSnapshot is what prevents this pause's count=2 - // checkpoint from restoring on the next resume. + // Revert from async suspend (local snapshot). Resume prefers a local + // snapshot over the external one, so clearing the local snapshot is what + // prevents this async suspend's count=2 checkpoint from restoring on the + // next resume. if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actorRef, Options: &ateapipb.SuspendOptions{Mode: ateapipb.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { t.Fatalf("failed to pause Actor: %v", err) } @@ -1196,7 +1197,7 @@ func revertActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Fatalf("RevertActor from SUSPENDED: got %v, want FailedPrecondition", err) } - // Final resume: if the pause checkpoint had been restored, the counter + // Final resume: if the local checkpoint had been restored, the counter // would advance from 2 to 3. Expecting 2 proves the external snapshot won. if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{Actor: actorRef}); err != nil { t.Fatalf("failed to resume Actor after paused revert: %v", err) @@ -1298,7 +1299,7 @@ func deletePausedActor(ctx context.Context, t *testing.T, clients *e2e.Clients, } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - // 3. Pausing the actor + // 3. Suspending the actor asynchronously t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, @@ -1308,7 +1309,7 @@ func deletePausedActor(ctx context.Context, t *testing.T, clients *e2e.Clients, } waitForActorPaused(ctx, t, clients, actorName) - // 4. Delete the paused actor + // 4. Delete the asynchronously suspended actor t.Logf("Deleting paused Actor %q...", actorName) if _, err := clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, @@ -1443,9 +1444,8 @@ func waitForActorState(ctx context.Context, t *testing.T, clients *e2e.Clients, waitForActorStateWithTimeout(ctx, t, clients, actorName, expectedState, 60*time.Second) } -// waitForActorPaused waits for the shape a pause leaves behind: SUSPENDED, -// holding a node-local snapshot and pinned to its node. A plain suspend -// leaves neither. +// waitForActorPaused waits for a SUSPENDED actor holding a node-local snapshot +// on its node. func waitForActorPaused(ctx context.Context, t *testing.T, clients *e2e.Clients, actorName string) { t.Helper() waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_SUSPENDED) diff --git a/internal/e2e/suites/demo/updatetemplate_test.go b/internal/e2e/suites/demo/updatetemplate_test.go index 488792402b..e886c81435 100644 --- a/internal/e2e/suites/demo/updatetemplate_test.go +++ b/internal/e2e/suites/demo/updatetemplate_test.go @@ -197,11 +197,11 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) t.Errorf("[after template update] expected %q (template B validating the preserved file), got response: %s", want, resp) } - // Pause under template B while status.snapshots still holds the last - // committed snapshot from template A. Resuming from pause restores the - // local checkpoint (which was captured under template B, since templates - // can only be updated while SUSPENDED) and preserves the in-memory counter - // when preferredFidelity is MEMORY. + // Suspend asynchronously under template B while status.snapshots still + // holds the last committed snapshot from template A. Resuming from the + // local snapshot restores the local checkpoint (which was captured under + // template B, since templates can only be updated while SUSPENDED) and + // preserves the in-memory counter when preferredFidelity is MEMORY. t.Logf("Pausing Actor %q under template B...", actorID) if _, err := clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{ Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, @@ -233,7 +233,7 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) if err != nil { t.Fatalf("failed to call actor after pause/resume under template B: %v", err) } - // The pause snapshot was captured under B, so it restores at its own + // The local snapshot was captured under B, so it restores at its own // fidelity: MEMORY keeps both counters, ROOTFS keeps the rootfs one, and // VOLUMES neither. wantMemAfterPause, wantRootfsAfterPause := 2, 2 @@ -324,7 +324,7 @@ func runUpdateTemplateTestCase(t *testing.T, fidelity ateapipb.SnapshotFidelity) // createUpdateTestTemplate creates a per-test WorkerPool plus a substrate // ActorTemplate copying the deployed counter fixture's resolved runtime, -// capturing at the scope under test on both pause and commit. +// capturing at the scope under test on suspend. func createUpdateTestTemplate(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj *e2e.Namespace, name, poolName, bucket string, fidelity ateapipb.SnapshotFidelity, modify func(*ateapipb.ActorTemplate)) *ateapipb.ActorTemplate { t.Helper() return e2e.CreateSubstrateCounterTemplate(ctx, t, clients, nsObj.Name, e2e.SubstrateTemplateOptions{ diff --git a/internal/objectstoreplugin/exchange/exchange_test.go b/internal/objectstoreplugin/exchange/exchange_test.go index 5776cc8a99..8bcdd954f0 100644 --- a/internal/objectstoreplugin/exchange/exchange_test.go +++ b/internal/objectstoreplugin/exchange/exchange_test.go @@ -186,7 +186,7 @@ func actorWithTag() *ateapipb.Actor { } } -// actorLocalOnly is a paused actor whose only snapshot is on node-1. +// actorLocalOnly is a suspended actor whose only snapshot is on node-1. func actorLocalOnly() *ateapipb.Actor { return &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name, Uid: uid}, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 070f24f250..c42a1b28a0 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -406,9 +406,9 @@ service Atelet { // PromoteSnapshot makes `snapshot` durable in object storage while keeping // its local copy on this node. Used when a snapshot needs to be in object - // storage (e.g. for tagging) without evicting the paused actor from the - // node. Waits for an in-flight detach, and fails for an actor running on - // this node. + // storage (e.g. for tagging) without evicting the suspended actor's local + // snapshot from the node. Waits for an in-flight detach, and fails for an + // actor running on this node. // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 861b3d7742..08dcf99ffc 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -381,9 +381,9 @@ type AteletClient interface { Restore(ctx context.Context, in *RestoreRequest, opts ...grpc.CallOption) (*RestoreResponse, error) // PromoteSnapshot makes `snapshot` durable in object storage while keeping // its local copy on this node. Used when a snapshot needs to be in object - // storage (e.g. for tagging) without evicting the paused actor from the - // node. Waits for an in-flight detach, and fails for an actor running on - // this node. + // storage (e.g. for tagging) without evicting the suspended actor's local + // snapshot from the node. Waits for an in-flight detach, and fails for an + // actor running on this node. // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. @@ -506,9 +506,9 @@ type AteletServer interface { Restore(context.Context, *RestoreRequest) (*RestoreResponse, error) // PromoteSnapshot makes `snapshot` durable in object storage while keeping // its local copy on this node. Used when a snapshot needs to be in object - // storage (e.g. for tagging) without evicting the paused actor from the - // node. Waits for an in-flight detach, and fails for an actor running on - // this node. + // storage (e.g. for tagging) without evicting the suspended actor's local + // snapshot from the node. Waits for an in-flight detach, and fails for an + // actor running on this node. // // Idempotent: once the snapshot is durable, atelet answers from the // manifest at `snapshot.object`, and fails if there is none. diff --git a/internal/resources/actorstate.go b/internal/resources/actorstate.go index e2c0e76b4a..fd5028a9ee 100644 --- a/internal/resources/actorstate.go +++ b/internal/resources/actorstate.go @@ -21,7 +21,7 @@ import "go.ate.dev/substrate/pkg/proto/ateapipb" // - RESUMING: a resume fetches the snapshot. // - RUNNING: the actor keeps the credentials its resume minted. // - SUSPENDING: a suspend uploads the snapshot. -// - SUSPENDED: a paused actor's background upload runs. +// - SUSPENDED: a background upload of a local snapshot runs. // // In every other state no node acts for the actor, so no node may hold a // credential for its snapshots. The mint's host check and the objectstore diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index b7dd5974f3..4e889509fc 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -739,9 +739,9 @@ type Snapshot struct { // +k8s:format=k8s-long-name Locality string `protobuf:"bytes,6,opt,name=locality,proto3" json:"locality,omitempty"` // durable_snapshot holds the durable storage representation of this snapshot. - // PENDING while the snapshot is uploaded, including a pause's snapshot that - // so far exists only on the node named by locality. COMPLETED once the - // upload is recorded. + // PENDING while the snapshot is uploaded, including an asynchronous suspend's + // snapshot that so far exists only on the node named by locality. COMPLETED + // once the upload is recorded. // // +k8s:optional DurableSnapshot *SnapshotStorage `protobuf:"bytes,7,opt,name=durable_snapshot,json=durableSnapshot,proto3" json:"durable_snapshot,omitempty"` @@ -2071,7 +2071,7 @@ type ActorStatus struct { // +k8s:listMapKey=generation Snapshots []*Snapshot `protobuf:"bytes,6,rep,name=snapshots,proto3" json:"snapshots,omitempty"` // last_assigned_generation tracks the last snapshot generation assigned for - // this Actor during a Suspend or Pause request. + // this Actor during a Suspend request. // Even during Revert, the generation is not decreased to keep track of snapshots // that have become stale and need to be cleaned up. // @@ -3040,7 +3040,7 @@ type SnapshotConfig struct { // +k8s:customValidation # Validate URI StorageLocation string `protobuf:"bytes,1,opt,name=storage_location,json=storageLocation,proto3" json:"storage_location,omitempty"` // preferred_fidelity is the highest fidelity the system packages when it - // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // snapshots the actor. // It is a best-effort ceiling, not a guarantee: the system never packages // more than this level, and it may package less when a layer cannot be // captured or is too expensive to capture at the time, for example memory diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 388882f45d..d574a2cf17 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -362,9 +362,9 @@ message Snapshot { string locality = 6; // durable_snapshot holds the durable storage representation of this snapshot. - // PENDING while the snapshot is uploaded, including a pause's snapshot that - // so far exists only on the node named by locality. COMPLETED once the - // upload is recorded. + // PENDING while the snapshot is uploaded, including an asynchronous suspend's + // snapshot that so far exists only on the node named by locality. COMPLETED + // once the upload is recorded. // // +k8s:optional SnapshotStorage durable_snapshot = 7; @@ -918,7 +918,7 @@ message ActorStatus { repeated Snapshot snapshots = 6; // last_assigned_generation tracks the last snapshot generation assigned for - // this Actor during a Suspend or Pause request. + // this Actor during a Suspend request. // Even during Revert, the generation is not decreased to keep track of snapshots // that have become stale and need to be cleaned up. // @@ -1237,7 +1237,7 @@ message SnapshotConfig { string storage_location = 1; // preferred_fidelity is the highest fidelity the system packages when it - // snapshots the actor, on pause (node-local) and on suspend (uploaded). + // snapshots the actor. // It is a best-effort ceiling, not a guarantee: the system never packages // more than this level, and it may package less when a layer cannot be // captured or is too expensive to capture at the time, for example memory @@ -2610,7 +2610,7 @@ service WorkerService { // The control plane mints only for the Worker that the Actor is assigned // to, and only to an atelet on that Worker's node. A request from any // other node or Worker gets NotFound; one from the assigned Worker with a - // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a SUSPENDED Actor // gets NotFound, since it has no Worker to bind the certificate to. rpc MintWorkerActorCertificate(MintWorkerActorCertificateRequest) returns (MintWorkerActorCertificateResponse); @@ -2633,7 +2633,7 @@ service WorkerService { // // This is a request, not a command. It runs the same suspend workflow // Control.SuspendActor runs, behind the same preconditions, so a proposal - // that races a resume, pause, or delete loses to it. + // that races a resume, suspend, or delete loses to it. // // atelet calls this with its own client certificate, as it does for // RegisterWorker, naming the Worker its caller proved itself to be. The diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index c21225257c..31545303e8 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -1925,7 +1925,7 @@ type WorkerServiceClient interface { // The control plane mints only for the Worker that the Actor is assigned // to, and only to an atelet on that Worker's node. A request from any // other node or Worker gets NotFound; one from the assigned Worker with a - // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a SUSPENDED Actor // gets NotFound, since it has no Worker to bind the certificate to. MintWorkerActorCertificate(ctx context.Context, in *MintWorkerActorCertificateRequest, opts ...grpc.CallOption) (*MintWorkerActorCertificateResponse, error) // MintWorkerActorJwt creates a Substrate-issued JWT that asserts a worker @@ -1946,7 +1946,7 @@ type WorkerServiceClient interface { // // This is a request, not a command. It runs the same suspend workflow // Control.SuspendActor runs, behind the same preconditions, so a proposal - // that races a resume, pause, or delete loses to it. + // that races a resume, suspend, or delete loses to it. // // atelet calls this with its own client certificate, as it does for // RegisterWorker, naming the Worker its caller proved itself to be. The @@ -2081,7 +2081,7 @@ type WorkerServiceServer interface { // The control plane mints only for the Worker that the Actor is assigned // to, and only to an atelet on that Worker's node. A request from any // other node or Worker gets NotFound; one from the assigned Worker with a - // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a paused Actor + // stale actor_uid gets Aborted. Unlike MintWorkerActorJwt, a SUSPENDED Actor // gets NotFound, since it has no Worker to bind the certificate to. MintWorkerActorCertificate(context.Context, *MintWorkerActorCertificateRequest) (*MintWorkerActorCertificateResponse, error) // MintWorkerActorJwt creates a Substrate-issued JWT that asserts a worker @@ -2102,7 +2102,7 @@ type WorkerServiceServer interface { // // This is a request, not a command. It runs the same suspend workflow // Control.SuspendActor runs, behind the same preconditions, so a proposal - // that races a resume, pause, or delete loses to it. + // that races a resume, suspend, or delete loses to it. // // atelet calls this with its own client certificate, as it does for // RegisterWorker, naming the Worker its caller proved itself to be. The