fix(server): stop orphaning the agent-device daemon after a missed health probe

apps/server/src/device/LocalDeviceHost.test.ts

@@ -51,6 +51,10 @@ const agentFixture = Effect.fn("agentFixture")(function* (
   const daemonStopRequested = yield* Deferred.make<ReadonlyArray<string>>();
   const stateDir = agentDeviceStateDir(path, path.join(baseDir, "userdata"));
   const daemonHealthy = yield* Ref.make(true);
+  const failedProbes = yield* Ref.make(0);
+  const commands: Array<string> = [];
+  const stopResult = yield* Ref.make(processOutput(0));
+  const daemonFileAtStop: Array<boolean> = [];
   const runs: Array<ProcessRunner.ProcessRunInput> = [];
   for (const [name, version, entry] of [
     ["expo-device-hub", DEVICE_HUB_VERSION, "dist/server/cli.mjs"],
@@ -105,17 +109,28 @@ const agentFixture = Effect.fn("agentFixture")(function* (
     Effect.provideService(ProcessRunner.ProcessRunner, {
       run: (input) => {
         runs.push(input);
+        if (input.args.includes("devices") || input.args.includes("daemon"))
+          commands.push(input.args.slice(1, 3).join(" "));
+        if (input.args.includes("daemon"))
+          return fs.exists(`${stateDir}/daemon.json`).pipe(
+            Effect.orDie,
+            Effect.tap((exists) => Effect.sync(() => daemonFileAtStop.push(exists))),
+            Effect.tap(() => Deferred.succeed(daemonStopRequested, input.args)),
+            Effect.andThen(Ref.get(stopResult)),
+          );
         return input.args.includes("devices")
           ? startup(stateDir)
-          : input.args.includes("daemon")
-            ? Deferred.succeed(daemonStopRequested, input.args).pipe(Effect.as(processOutput(0)))
-            : Effect.succeed(input.args.includes("-list-avds") ? avdOutput : processOutput(0));
+          : Effect.succeed(input.args.includes("-list-avds") ? avdOutput : processOutput(0));
       },
     }),
     Effect.provideService(
       HttpClient.HttpClient,
       HttpClient.make((request) =>
-        Ref.get(daemonHealthy).pipe(
+        Effect.all([
+          Ref.get(daemonHealthy),
+          Ref.getAndUpdate(failedProbes, (remaining) => Math.max(0, remaining - 1)),
+        ]).pipe(
+          Effect.map(([daemonUp, failing]) => daemonUp && failing === 0),
           Effect.map((healthy) =>
             HttpClientResponse.fromWeb(
               request,
@@ -129,15 +144,28 @@ const agentFixture = Effect.fn("agentFixture")(function* (
     ),
   );
   yield* host.ensureReady(() => Effect.void);
-  return { host, fs, baseDir, stateDir, hub, daemonHealthy, daemonStopRequested, runs };
+  return {
+    host,
+    fs,
+    baseDir,
+    stateDir,
+    hub,
+    daemonHealthy,
+    daemonStopRequested,
+    failedProbes,
+    commands,
+    stopResult,
+    daemonFileAtStop,
+    runs,
+  };
 });
 
 describe("AgentDevice startup", () => {
   it.effect("restarts a cached daemon that no longer answers its health check", () =>
     Effect.gen(function* () {
       const fs = yield* FileSystem.FileSystem;
       let attempts = 0;
-      const { host, daemonHealthy } = yield* agentFixture((stateDir) =>
+      const { host, daemonHealthy, commands, daemonFileAtStop } = yield* agentFixture((stateDir) =>
         Effect.gen(function* () {
           attempts++;
           yield* fs
@@ -158,13 +186,68 @@ describe("AgentDevice startup", () => {
       yield* Ref.set(daemonHealthy, false);
       const restarted = yield* host.ensureAgentReady(() => Effect.void);
       expect(attempts).toBe(2);
+      // The old daemon is stopped while daemon.json still identifies it.
+      expect(commands).toEqual(["devices --json", "daemon stop", "devices --json"]);
+      expect(daemonFileAtStop).toEqual([true]);
       expect(restarted.agentDevice).toMatchObject({
         baseUrl: "http://127.0.0.1:12347",
         token: "token-2",
       });
     }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
   );
 
+  it.effect("still restarts an unresponsive daemon when its stop times out", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      let attempts = 0;
+      const { host, daemonHealthy, commands, stopResult } = yield* agentFixture((stateDir) =>
+        Effect.gen(function* () {
+          attempts++;
+          yield* fs
+            .writeFileString(
+              `${stateDir}/daemon.json`,
+              JSON.stringify({ httpPort: 12345 + attempts, token: `token-${attempts}` }),
+            )
+            .pipe(Effect.orDie);
+          return processOutput(0);
+        }),
+      );
+      yield* host.ensureAgentReady(() => Effect.void);
+      yield* Ref.set(daemonHealthy, false);
+      yield* Ref.set(stopResult, processOutput(0, "", "", true));
+
+      const restarted = yield* host.ensureAgentReady(() => Effect.void);
+      expect(commands).toEqual(["devices --json", "daemon stop", "devices --json"]);
+      expect(restarted.agentDevice).toMatchObject({ token: "token-2" });
+    }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
+  );
+
+  it.effect("keeps a busy daemon that misses a health probe", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      let attempts = 0;
+      const { host, failedProbes, commands } = yield* agentFixture((stateDir) =>
+        Effect.gen(function* () {
+          attempts++;
+          yield* fs
+            .writeFileString(
+              `${stateDir}/daemon.json`,
+              JSON.stringify({ httpPort: 12345 + attempts, token: `token-${attempts}` }),
+            )
+            .pipe(Effect.orDie);
+          return processOutput(0);
+        }),
+      );
+      const first = yield* host.ensureAgentReady(() => Effect.void);
+
+      yield* Ref.set(failedProbes, 1);
+      const again = yield* host.ensureAgentReady(() => Effect.void);
+      expect(again.agentDevice).toEqual(first.agentDevice);
+      expect(attempts).toBe(1);
+      expect(commands).toEqual(["devices --json"]);
+    }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
+  );
+
   it.effect("reports an incompatible lock immediately and retains it for manual repair", () =>
     Effect.gen(function* () {
       const result = processOutput(

apps/server/src/device/LocalDeviceHost.ts

@@ -55,6 +55,11 @@ import {
 const HUB_READY_TIMEOUT_MS = 30_000;
 const DAEMON_READY_TIMEOUT_MS = 30_000;
 const DAEMON_POLL_MS = 100;
+const DAEMON_HEALTH_TIMEOUT_MS = 3_000;
+const DAEMON_HEALTH_ATTEMPTS = 3;
+// agent-device waits up to 10 s after SIGTERM and 2 s after SIGKILL before it
+// reports whether the daemon exited.
+const DAEMON_RECOVERY_STOP_TIMEOUT = Duration.seconds(15);
 const HUB_RESTART_STABLE_UPTIME_MS = 60_000;
 const HUB_RESTART_MAX_DELAY_MS = 30_000;
 
@@ -553,17 +558,35 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
       entryPath: agentTool.entryPath,
     });
     if (existing._tag === "Some") {
-      const alive = yield* HttpClient.withScope(httpClient)
+      const probe = HttpClient.withScope(httpClient)
         .get(`http://127.0.0.1:${existing.value.httpPort}/health`)
         .pipe(
-          Effect.timeout(Duration.seconds(2)),
           Effect.flatMap((response) =>
             response.arrayBuffer.pipe(Effect.as(response.status === 200)),
           ),
+          Effect.timeout(Duration.millis(DAEMON_HEALTH_TIMEOUT_MS)),
           Effect.scoped,
           Effect.orElseSucceed(() => false),
         );
-      if (alive) return toEndpoint(existing.value);
+      // A daemon busy with a long command can miss one probe; restarting it then
+      // would race its live lock and wedge every later start.
+      for (let attempt = 1; attempt <= DAEMON_HEALTH_ATTEMPTS; attempt++) {
+        if (yield* probe) return toEndpoint(existing.value);
+      }
+      // Stop through agent-device while daemon.json still names the process; once
+      // the file is gone it reports "not-running" and the daemon keeps daemon.lock.
+      const stopped = yield* runAgentDeviceDaemonStop(
+        { entryPath: agentTool.entryPath, nodePath },
+        DAEMON_RECOVERY_STOP_TIMEOUT,
+      ).pipe(Effect.result);
+      if (stopped._tag === "Failure" || stopped.success.timedOut || stopped.success.code !== 0) {
+        yield* Effect.logWarning(
+          "Could not stop an unresponsive agent-device daemon",
+          stopped._tag === "Failure"
+            ? { cause: stopped.failure }
+            : { code: stopped.success.code, timedOut: stopped.success.timedOut },
+        );
+      }
       yield* fs.remove(daemonFilePath(), { force: true }).pipe(Effect.ignore);
     }
     // No `daemon start` exists; `devices` is the cheapest command that starts and awaits it.
@@ -626,6 +649,24 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
     }
   });
 
+  const runAgentDeviceDaemonStop = (
+    agentTool: { readonly entryPath: string; readonly nodePath: string },
+    timeout: Duration.Duration,
+  ) =>
+    runner.run({
+      command: agentTool.nodePath,
+      args: [
+        agentTool.entryPath,
+        "daemon",
+        "stop",
+        "--state-dir",
+        agentDeviceStateDir(path, config.stateDir),
+      ],
+      env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" },
+      timeout,
+      timeoutBehavior: "timedOutResult",
+    });
+
   // The daemon may predate this process (an earlier server was killed, possibly on an
   // older pin), so stop it with the CLI of its recorded version when that is installed.
   const stopAgentDeviceDaemon = Effect.gen(function* () {
@@ -646,19 +687,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () {
     const nodePath = yield* resolveNodeExecutable("Local device support", hostEnvironment).pipe(
       Effect.provideService(HostProcess.Platform, hostPlatform),
     );
-    yield* runner.run({
-      command: nodePath,
-      args: [
-        entryPath,
-        "daemon",
-        "stop",
-        "--state-dir",
-        agentDeviceStateDir(path, config.stateDir),
-      ],
-      env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" },
-      timeout: Duration.seconds(10),
-      timeoutBehavior: "timedOutResult",
-    });
+    yield* runAgentDeviceDaemonStop({ entryPath, nodePath }, Duration.seconds(10));
   }).pipe(
     Effect.provideService(FileSystem.FileSystem, fs),
     Effect.provideService(Path.Path, path),