From d49ba94c6189d9aa04b0081116ffa1d5a7be8b24 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 18:53:27 +0000 Subject: [PATCH 1/5] test(server): reproduce launch agent left pending after bootstrap (#29522) These tests reproduce #29522: launchd can load a bootstrapped launch agent without spawning it, and install, restart and withStopped then report success while the service stays down. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.test.ts | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 8fcb026d4a45d..3c50611cc6a53 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -110,11 +110,16 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( linger: string; enabled: boolean; active: boolean; + // launchd can load a bootstrapped job yet leave its RunAtLoad spawn pending. + bootstrapSpawns: boolean; + launchdRunning: boolean; } = { failCommand: undefined, linger: "yes", enabled: true, active: true, + bootstrapSpawns: true, + launchdRunning: false, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -131,6 +136,16 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( if (!failed && command === "systemctl --user restart t3code.service") control.active = true; // A timed-out stop may still stop the service. if (command === "systemctl --user stop t3code.service") control.active = false; + if (command.startsWith("launchctl bootout ")) control.launchdRunning = false; + if (!failed && command.startsWith("launchctl bootstrap ")) + control.launchdRunning = control.bootstrapSpawns; + if ( + !failed && + input.command === "launchctl" && + input.args[0] === "kickstart" && + input.args.at(-1) === "gui/501/com.t3tools.t3code.service" + ) + control.launchdRunning = true; if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || @@ -906,4 +921,23 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect((yield* service.status).current).toBe(true); }), ); + + // regression: #29522 + it.effect.each(["install", "restart", "withStopped"] as const)( + "runs a launch agent that launchd leaves pending after bootstrap via %s", + (operation) => + Effect.gen(function* () { + const { service, control } = yield* makeHarness("darwin"); + yield* service.install(); + control.bootstrapSpawns = false; + + yield* operation === "install" + ? service.install() + : operation === "restart" + ? service.restart + : service.withStopped(Effect.void); + + expect(control.launchdRunning, "launch agent is loaded but not running").toBe(true); + }), + ); }); From 32ed96405a9c41ca2fbd48084c574bae2d7268fd Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 18:56:48 +0000 Subject: [PATCH 2/5] fix(server): start launch agents that launchd leaves pending after bootstrap launchd can load a bootstrapped RunAtLoad/KeepAlive job without spawning it, so install, restart, update and the withStopped restore reported success while the service stayed down. Run a plain `launchctl kickstart` after every bootstrap; without -k it starts a loaded job that is not running and leaves a running one alone. Fixes pingdotgg/t3code#29522 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.test.ts | 1 + apps/server/src/cloud/bootService.ts | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 3c50611cc6a53..4c8e80f0d0b74 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -440,6 +440,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { : [ "launchctl bootout --wait gui/501/com.t3tools.t3code.service", `launchctl bootstrap gui/501 ${plan.unitPath}`, + "launchctl kickstart gui/501/com.t3tools.t3code.service", ], ); }), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index efa18413d979c..e286bab166183 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -301,7 +301,7 @@ function launchdManager(input: { ); const domainTarget = `gui/${input.uid}`; const serviceTarget = `${domainTarget}/${BOOT_SERVICE_LAUNCHD_LABEL}`; - // Unloaded jobs may reject bootout/enable. Bootstrap starts RunAtLoad/KeepAlive jobs; kickstart would restart them. + // Unloaded jobs may reject bootout/enable. Bootstrap can leave RunAtLoad jobs pending; plain kickstart starts them. return { kind: "launchd", unitPath, @@ -333,6 +333,11 @@ function launchdManager(input: { command: "launchctl", args: ["bootstrap", domainTarget, unitPath], }, + { + step: "starting the service", + command: "launchctl", + args: ["kickstart", serviceTarget], + }, ], restart: [ { @@ -340,6 +345,11 @@ function launchdManager(input: { command: "launchctl", args: ["bootstrap", domainTarget, unitPath], }, + { + step: "restarting the service after a failed update", + command: "launchctl", + args: ["kickstart", serviceTarget], + }, ], // Persisted launchctl disable overrides would block reinstall. deactivate: [ From be6b738faaefcde6037c8bc83ef83b7a3865eec8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 19:04:46 +0000 Subject: [PATCH 3/5] fix(server): name the launch agent kickstart in boot service failures Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index e286bab166183..ff8a90e1465cf 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -334,7 +334,7 @@ function launchdManager(input: { args: ["bootstrap", domainTarget, unitPath], }, { - step: "starting the service", + step: "starting the launch agent", command: "launchctl", args: ["kickstart", serviceTarget], }, @@ -346,7 +346,7 @@ function launchdManager(input: { args: ["bootstrap", domainTarget, unitPath], }, { - step: "restarting the service after a failed update", + step: "starting the launch agent after a failed update", command: "launchctl", args: ["kickstart", serviceTarget], }, From f7e0be36894f31a08d5f932fba298b7ab0fbff63 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 23:38:45 +0000 Subject: [PATCH 4/5] fix(server): start a still-loaded launch agent when recovery bootstrap is rejected AI-assisted; verified with the tests and tools listed above. --- apps/server/src/cloud/bootService.test.ts | 45 +++++++++++++++-------- apps/server/src/cloud/bootService.ts | 3 ++ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 4c8e80f0d0b74..01073ad9567d3 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -112,6 +112,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( active: boolean; // launchd can load a bootstrapped job yet leave its RunAtLoad spawn pending. bootstrapSpawns: boolean; + launchdLoaded: boolean; launchdRunning: boolean; } = { failCommand: undefined, @@ -119,6 +120,7 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( enabled: true, active: true, bootstrapSpawns: true, + launchdLoaded: false, launchdRunning: false, }; const runner = ProcessRunner.ProcessRunner.of({ @@ -129,23 +131,27 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands.push(command); invocations.push(input); timeouts.set(command, input.timeout); - const failed = command === control.failCommand; + const launchdVerb = input.command === "launchctl" ? input.args[0] : undefined; + // launchd rejects bootstrap of a loaded job and kickstart of an unloaded one. + const failed = + command === control.failCommand || + (launchdVerb === "bootstrap" && control.launchdLoaded) || + (launchdVerb === "kickstart" && !control.launchdLoaded); if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; if (!failed && command === "systemctl --user restart t3code.service") control.active = true; // A timed-out stop may still stop the service. if (command === "systemctl --user stop t3code.service") control.active = false; - if (command.startsWith("launchctl bootout ")) control.launchdRunning = false; - if (!failed && command.startsWith("launchctl bootstrap ")) + if (launchdVerb === "bootout") { + control.launchdRunning = false; + if (!failed) control.launchdLoaded = false; + } + if (!failed && launchdVerb === "bootstrap") { + control.launchdLoaded = true; control.launchdRunning = control.bootstrapSpawns; - if ( - !failed && - input.command === "launchctl" && - input.args[0] === "kickstart" && - input.args.at(-1) === "gui/501/com.t3tools.t3code.service" - ) - control.launchdRunning = true; + } + if (!failed && launchdVerb === "kickstart") control.launchdRunning = true; if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || @@ -890,6 +896,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { Effect.gen(function* () { const { service, control } = yield* makeHarness("darwin"); yield* service.install(); + control.launchdLoaded = false; control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; yield* service.install(); @@ -924,7 +931,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ); // regression: #29522 - it.effect.each(["install", "restart", "withStopped"] as const)( + it.effect.each(["install", "restart", "withStopped", "withStopped after a failed stop"] as const)( "runs a launch agent that launchd leaves pending after bootstrap via %s", (operation) => Effect.gen(function* () { @@ -932,11 +939,17 @@ it.layer(NodeServices.layer)("boot service install", (it) => { yield* service.install(); control.bootstrapSpawns = false; - yield* operation === "install" - ? service.install() - : operation === "restart" - ? service.restart - : service.withStopped(Effect.void); + if (operation === "withStopped after a failed stop") { + // The bootout timed out with the job still loaded, so recovery's bootstrap is rejected. + control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; + yield* service.withStopped(Effect.void).pipe(Effect.flip); + } else { + yield* operation === "install" + ? service.install() + : operation === "restart" + ? service.restart + : service.withStopped(Effect.void); + } expect(control.launchdRunning, "launch agent is loaded but not running").toBe(true); }), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index ff8a90e1465cf..40e669f5a8fc8 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -339,11 +339,14 @@ function launchdManager(input: { args: ["kickstart", serviceTarget], }, ], + // A failed stop can leave the job loaded, so bootstrap may be rejected; kickstart still + // starts it, and fails on its own when bootstrap left nothing loaded. restart: [ { step: "restarting the service after a failed update", command: "launchctl", args: ["bootstrap", domainTarget, unitPath], + optional: true, }, { step: "starting the launch agent after a failed update", From dfd1b146ee8f1490460d30ce002f50ae2770b16a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:08:22 +0000 Subject: [PATCH 5/5] test(server): keep the non-overlapping launch agent pending cases Co-Authored-By: Claude Opus 5.5 --- apps/server/src/cloud/bootService.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 5706ad2713329..33b95e7bc9d5e 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1008,7 +1008,7 @@ it.layer(NodeServices.layer)("boot service install", (it) => { ); // regression: #29522 - it.effect.each(["install", "restart", "withStopped", "withStopped after a failed stop"] as const)( + it.effect.each(["install", "withStopped after a failed stop"] as const)( "runs a launch agent that launchd leaves pending after bootstrap via %s", (operation) => Effect.gen(function* () { @@ -1016,16 +1016,12 @@ it.layer(NodeServices.layer)("boot service install", (it) => { yield* service.install(); control.bootstrapSpawns = false; - if (operation === "withStopped after a failed stop") { + if (operation === "install") { + yield* service.install(); + } else { // The bootout timed out with the job still loaded, so recovery's bootstrap is rejected. control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; yield* service.withStopped(Effect.void).pipe(Effect.flip); - } else { - yield* operation === "install" - ? service.install() - : operation === "restart" - ? service.restart - : service.withStopped(Effect.void); } expect(control.launchdRunning, "launch agent is loaded but not running").toBe(true);