fix(server): retain the runtime used by the service launcher

apps/server/src/cloud/bootService.test.ts

@@ -300,9 +300,12 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
       const { service, fs, statePath, timeouts, runtime } = yield* makeHarness();
       const plan = yield* service.install();
 
+      // The unit runs the launcher from this runtime even after remote updates move
+      // activeVersion on, so state has to record it for anything that cleans up runtimes.
       expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
         protocol: SERVICE_LAUNCHER_PROTOCOL,
         activeVersion: "1.2.3",
+        launcherVersion: "1.2.3",
       });
       expect(plan.program).toEqual([runtime.entryPath, "__service-launcher"]);
       expect(yield* fs.readFileString(plan.unitPath)).toContain(
@@ -468,6 +471,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
         expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
           protocol: SERVICE_LAUNCHER_PROTOCOL,
           activeVersion: "1.2.4",
+          launcherVersion: "1.2.4",
         });
         expect(yield* fs.readFileString(plan.unitPath)).toContain("versions/1.2.4/t3");
         expect(
@@ -763,6 +767,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => {
       expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({
         protocol: SERVICE_LAUNCHER_PROTOCOL,
         activeVersion: "1.2.3",
+        launcherVersion: "1.2.3",
       });
       expect(yield* fs.readFileString(plan.unitPath)).toContain(
         `    <string>${runtime.entryPath}</string>\n    <string>__service-launcher</string>`,

apps/server/src/cloud/bootService.ts

@@ -869,6 +869,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: {
           {
             protocol: SERVICE_LAUNCHER_PROTOCOL,
             activeVersion: input.cliVersion,
+            launcherVersion: input.cliVersion,
           } satisfies ServiceState,
           null,
           2,

apps/server/src/cloud/pinnedRuntime.test.ts

@@ -428,6 +428,35 @@ it.layer(NodeServices.layer)("prunePinnedRuntimes", (it) => {
     }),
   );
 
+  it.effect("keeps the runtime the service state records as the launcher", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const path = yield* Path.Path;
+      const { baseDir } = yield* setup(["0.1.1", "0.1.2", "0.1.3"]);
+      // Installed from 0.1.1, then updated remotely twice. The service is stopped, so nothing
+      // runs 0.1.1, but its unit will start the launcher from it.
+      yield* fs.writeFileString(
+        path.join(baseDir, "runtime", "service-state.json"),
+        JSON.stringify({
+          protocol: SERVICE_LAUNCHER_PROTOCOL,
+          activeVersion: "0.1.3",
+          launcherVersion: "0.1.1",
+          update: { id: "u", fromVersion: "0.1.2", targetVersion: "0.1.3", status: "committed" },
+        }),
+      );
+
+      const removed = yield* prunePinnedRuntimes({ baseDir, keep: ["0.1.3"] }).pipe(
+        Effect.provideService(
+          ProcessRunner.ProcessRunner,
+          lsofRunner("p1\nftxt\nn/sbin/launchd\n"),
+        ),
+        Effect.provideService(HostProcess.Platform, "darwin"),
+        Effect.provideService(HostProcess.Environment, { T3CODE_INSTALL_BIN_DIR: baseDir }),
+      );
+      assert.deepEqual(removed, []);
+    }),
+  );
+
   it.effect("matches a Windows shim whatever its casing", () =>
     Effect.gen(function* () {
       const fs = yield* FileSystem.FileSystem;

apps/server/src/cloud/pinnedRuntime.ts

@@ -551,6 +551,7 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun
     const state = parseServiceState(stateText.value);
     if (state === undefined) return pruned;
     keep.add(state.activeVersion);
+    if (state.launcherVersion !== undefined) keep.add(state.launcherVersion);
     if (state.update !== undefined) {
       keep.add(state.update.fromVersion);
       keep.add(state.update.targetVersion);
@@ -600,9 +601,9 @@ const selectUnusedRuntimes = Effect.fn("cloud.pinned_runtime.select_unused")(fun
 /**
  * Every update publishes a new immutable runtime, so without pruning they pile up. Removes
  * completed runtimes nothing can start again and keeps: the caller's versions, everything the
- * service state references (active, rollback, pending), and runtimes that a running process
- * executes, a `t3` launcher points at, or `references` (such as the service unit) name, plus
- * anything published within the last hour. Staging directories, symlinks, and unexpected entries
+ * service state references (active, launcher, rollback, pending), and runtimes that a running
+ * process executes, a `t3` launcher points at, or `references` (such as the service unit) name,
+ * plus anything published within the last hour. Staging directories, symlinks, and unexpected entries
  * are never touched. Returns the versions removed.
  */
 export const prunePinnedRuntimes = Effect.fn("cloud.pinned_runtime.prune")(function* (input: {

apps/server/src/cloud/serviceProtocol.ts

@@ -29,6 +29,10 @@ export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome
 export interface ServiceState {
   readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL;
   readonly activeVersion: string;
+  /** The runtime the installed unit starts the launcher from. Remote updates move
+      `activeVersion` on without touching the unit, so cleanup has to keep this one.
+      Absent in state written by older installers, and launchers before it drop it. */
+  readonly launcherVersion?: string;
   readonly update?: ServiceUpdateRecord;
 }
 
@@ -152,6 +156,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
     value.protocol !== SERVICE_LAUNCHER_PROTOCOL ||
     typeof value.activeVersion !== "string" ||
     !isExactServiceVersion(value.activeVersion) ||
+    (value.launcherVersion !== undefined &&
+      (typeof value.launcherVersion !== "string" ||
+        !isExactServiceVersion(value.launcherVersion))) ||
     (value.update !== undefined && update === undefined) ||
     (update !== undefined &&
       compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) ||
@@ -165,6 +172,9 @@ export function decodeServiceState(value: unknown): ServiceState | undefined {
   return {
     protocol: SERVICE_LAUNCHER_PROTOCOL,
     activeVersion: value.activeVersion,
+    ...(typeof value.launcherVersion === "string"
+      ? { launcherVersion: value.launcherVersion }
+      : {}),
     ...(update === undefined ? {} : { update }),
   };
 }

apps/server/src/serviceLauncher.test.ts

@@ -253,17 +253,20 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
     }),
   );
 
-  it.effect("commits only after the trial reports prepared", () =>
-    Effect.gen(function* () {
-      const fs = yield* FileSystem.FileSystem;
-      const path = yield* Path.Path;
-      const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
-      const statePath = path.join(root, "runtime", "service-state.json");
-      const databasePath = path.join(root, "userdata", "state.sqlite");
-      yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
-      yield* fs.writeFileString(databasePath, "before trial");
-      const encodedDatabasePath = JSON.stringify(databasePath);
-      const childSource = `
+  // `t3 update` without a restart repoints the unit while the launcher keeps running.
+  it.effect.each([undefined, "1.1.0"])(
+    "commits only after the trial reports prepared (unit repointed to: %s)",
+    (repointedLauncher) =>
+      Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const path = yield* Path.Path;
+        const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" });
+        const statePath = path.join(root, "runtime", "service-state.json");
+        const databasePath = path.join(root, "userdata", "state.sqlite");
+        yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
+        yield* fs.writeFileString(databasePath, "before trial");
+        const encodedDatabasePath = JSON.stringify(databasePath);
+        const childSource = `
 const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
 if (context.update?.status === "pending") {
   process.send({ type: "prepared", updateId: context.update.id });
@@ -277,33 +280,48 @@ if (context.update?.status === "pending") {
   process.exit(0);
 }
 `;
-      for (const version of ["1.0.0", "1.1.0"]) {
-        yield* writeFakeRuntime(
-          fs,
-          path,
-          path.join(root, "runtime", "versions", version),
-          childSource,
+        for (const version of ["1.0.0", "1.1.0"]) {
+          yield* writeFakeRuntime(
+            fs,
+            path,
+            path.join(root, "runtime", "versions", version),
+            childSource,
+          );
+        }
+        yield* Effect.promise(() =>
+          writeServiceState(statePath, {
+            protocol: SERVICE_LAUNCHER_PROTOCOL,
+            activeVersion: "1.0.0",
+            launcherVersion: "1.0.0",
+          }),
         );
-      }
-      yield* Effect.promise(() =>
-        writeServiceState(statePath, {
-          protocol: SERVICE_LAUNCHER_PROTOCOL,
-          activeVersion: "1.0.0",
-        }),
-      );
 
-      const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
-      yield* Effect.promise(() =>
-        launcher.run().then(
-          () => Promise.reject(new Error("launcher unexpectedly completed")),
-          () => Promise.resolve(),
-        ),
-      );
+        const launcher = new Launcher(
+          root,
+          yield* Effect.promise(() => readServiceState(statePath)),
+        );
+        if (repointedLauncher !== undefined) {
+          yield* Effect.promise(() =>
+            writeServiceState(statePath, {
+              protocol: SERVICE_LAUNCHER_PROTOCOL,
+              activeVersion: "1.0.0",
+              launcherVersion: repointedLauncher,
+            }),
+          );
+        }
+        yield* Effect.promise(() =>
+          launcher.run().then(
+            () => Promise.reject(new Error("launcher unexpectedly completed")),
+            () => Promise.resolve(),
+          ),
+        );
 
-      const state = yield* Effect.promise(() => readServiceState(statePath));
-      assert.equal(state.activeVersion, "1.1.0");
-      assert.equal(state.update?.status, "committed");
-    }),
+        const state = yield* Effect.promise(() => readServiceState(statePath));
+        assert.equal(state.activeVersion, "1.1.0");
+        assert.equal(state.update?.status, "committed");
+        // The service unit still starts the launcher from 1.0.0 unless it was repointed.
+        assert.equal(state.launcherVersion, repointedLauncher ?? "1.0.0");
+      }),
   );
 
   it.effect("rolls back invalid trials with the startup environment", () =>

apps/server/src/serviceLauncher.ts

@@ -557,6 +557,24 @@ export class Launcher {
     this.#timer = undefined;
   }
 
+  async #writeState(next: ServiceState): Promise<void> {
+    // `t3 update` without a restart repoints the unit and records its launcher
+    // runtime while this launcher keeps running, so what the file says wins.
+    const current = await NodeFSP.readFile(this.#statePath, "utf8").then(
+      parseServiceState,
+      () => undefined,
+    );
+    const { launcherVersion } = current ?? next;
+    const state: ServiceState = {
+      protocol: next.protocol,
+      activeVersion: next.activeVersion,
+      ...(launcherVersion === undefined ? {} : { launcherVersion }),
+      ...(next.update === undefined ? {} : { update: next.update }),
+    };
+    await writeServiceState(this.#statePath, state);
+    this.#state = state;
+  }
+
   async #recover(): Promise<void> {
     // A fresh launcher means servers are running again: any stop marker from
     // a previous explicit stop is stale and must not make a future update
@@ -724,8 +742,7 @@ export class Launcher {
       status: "pending",
     };
     const next: ServiceState = { ...this.#state, update: pending };
-    await writeServiceState(this.#statePath, next);
-    this.#state = next;
+    await this.#writeState(next);
     await sendMessage(child.process, { type: "update-accepted", updateId: pending.id });
     this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS);
   }
@@ -762,8 +779,7 @@ export class Launcher {
       activeVersion: pending.targetVersion,
       update: committed,
     };
-    await writeServiceState(this.#statePath, next);
-    this.#state = next;
+    await this.#writeState(next);
     child.role = "active";
     await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined);
     await sendMessage(child.process, { type: "committed", updateId: committed.id });
@@ -826,8 +842,7 @@ export class Launcher {
         activeVersion: pending.fromVersion,
         update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
       };
-      await writeServiceState(this.#statePath, next);
-      this.#state = next;
+      await this.#writeState(next);
       await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
     }
     // Signal listeners alone do not keep Node alive.
@@ -854,8 +869,7 @@ export class Launcher {
       activeVersion: pending.fromVersion,
       update: outcome,
     };
-    await writeServiceState(this.#statePath, next);
-    this.#state = next;
+    await this.#writeState(next);
     await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
     await this.#startChild(next.activeVersion, "active", outcome);
   }