diff --git a/apps/server/src/device/LocalDeviceHost.test.ts b/apps/server/src/device/LocalDeviceHost.test.ts index 5f7cef54f5277..79bcc109c4136 100644 --- a/apps/server/src/device/LocalDeviceHost.test.ts +++ b/apps/server/src/device/LocalDeviceHost.test.ts @@ -51,6 +51,10 @@ const agentFixture = Effect.fn("agentFixture")(function* ( const daemonStopRequested = yield* Deferred.make>(); const stateDir = agentDeviceStateDir(path, path.join(baseDir, "userdata")); const daemonHealthy = yield* Ref.make(true); + const failedProbes = yield* Ref.make(0); + const commands: Array = []; + const stopResult = yield* Ref.make(processOutput(0)); + const daemonFileAtStop: Array = []; const runs: Array = []; for (const [name, version, entry] of [ ["expo-device-hub", DEVICE_HUB_VERSION, "dist/server/cli.mjs"], @@ -105,17 +109,28 @@ const agentFixture = Effect.fn("agentFixture")(function* ( Effect.provideService(ProcessRunner.ProcessRunner, { run: (input) => { runs.push(input); + if (input.args.includes("devices") || input.args.includes("daemon")) + commands.push(input.args.slice(1, 3).join(" ")); + if (input.args.includes("daemon")) + return fs.exists(`${stateDir}/daemon.json`).pipe( + Effect.orDie, + Effect.tap((exists) => Effect.sync(() => daemonFileAtStop.push(exists))), + Effect.tap(() => Deferred.succeed(daemonStopRequested, input.args)), + Effect.andThen(Ref.get(stopResult)), + ); return input.args.includes("devices") ? startup(stateDir) - : input.args.includes("daemon") - ? Deferred.succeed(daemonStopRequested, input.args).pipe(Effect.as(processOutput(0))) - : Effect.succeed(input.args.includes("-list-avds") ? avdOutput : processOutput(0)); + : Effect.succeed(input.args.includes("-list-avds") ? avdOutput : processOutput(0)); }, }), Effect.provideService( HttpClient.HttpClient, HttpClient.make((request) => - Ref.get(daemonHealthy).pipe( + Effect.all([ + Ref.get(daemonHealthy), + Ref.getAndUpdate(failedProbes, (remaining) => Math.max(0, remaining - 1)), + ]).pipe( + Effect.map(([daemonUp, failing]) => daemonUp && failing === 0), Effect.map((healthy) => HttpClientResponse.fromWeb( request, @@ -129,7 +144,20 @@ const agentFixture = Effect.fn("agentFixture")(function* ( ), ); yield* host.ensureReady(() => Effect.void); - return { host, fs, baseDir, stateDir, hub, daemonHealthy, daemonStopRequested, runs }; + return { + host, + fs, + baseDir, + stateDir, + hub, + daemonHealthy, + daemonStopRequested, + failedProbes, + commands, + stopResult, + daemonFileAtStop, + runs, + }; }); describe("AgentDevice startup", () => { @@ -137,7 +165,7 @@ describe("AgentDevice startup", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; let attempts = 0; - const { host, daemonHealthy } = yield* agentFixture((stateDir) => + const { host, daemonHealthy, commands, daemonFileAtStop } = yield* agentFixture((stateDir) => Effect.gen(function* () { attempts++; yield* fs @@ -158,6 +186,9 @@ describe("AgentDevice startup", () => { yield* Ref.set(daemonHealthy, false); const restarted = yield* host.ensureAgentReady(() => Effect.void); expect(attempts).toBe(2); + // The old daemon is stopped while daemon.json still identifies it. + expect(commands).toEqual(["devices --json", "daemon stop", "devices --json"]); + expect(daemonFileAtStop).toEqual([true]); expect(restarted.agentDevice).toMatchObject({ baseUrl: "http://127.0.0.1:12347", token: "token-2", @@ -165,6 +196,58 @@ describe("AgentDevice startup", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("still restarts an unresponsive daemon when its stop times out", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let attempts = 0; + const { host, daemonHealthy, commands, stopResult } = yield* agentFixture((stateDir) => + Effect.gen(function* () { + attempts++; + yield* fs + .writeFileString( + `${stateDir}/daemon.json`, + JSON.stringify({ httpPort: 12345 + attempts, token: `token-${attempts}` }), + ) + .pipe(Effect.orDie); + return processOutput(0); + }), + ); + yield* host.ensureAgentReady(() => Effect.void); + yield* Ref.set(daemonHealthy, false); + yield* Ref.set(stopResult, processOutput(0, "", "", true)); + + const restarted = yield* host.ensureAgentReady(() => Effect.void); + expect(commands).toEqual(["devices --json", "daemon stop", "devices --json"]); + expect(restarted.agentDevice).toMatchObject({ token: "token-2" }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps a busy daemon that misses a health probe", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + let attempts = 0; + const { host, failedProbes, commands } = yield* agentFixture((stateDir) => + Effect.gen(function* () { + attempts++; + yield* fs + .writeFileString( + `${stateDir}/daemon.json`, + JSON.stringify({ httpPort: 12345 + attempts, token: `token-${attempts}` }), + ) + .pipe(Effect.orDie); + return processOutput(0); + }), + ); + const first = yield* host.ensureAgentReady(() => Effect.void); + + yield* Ref.set(failedProbes, 1); + const again = yield* host.ensureAgentReady(() => Effect.void); + expect(again.agentDevice).toEqual(first.agentDevice); + expect(attempts).toBe(1); + expect(commands).toEqual(["devices --json"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("reports an incompatible lock immediately and retains it for manual repair", () => Effect.gen(function* () { const result = processOutput( diff --git a/apps/server/src/device/LocalDeviceHost.ts b/apps/server/src/device/LocalDeviceHost.ts index e657d0dcdfc13..ce7f4cd2bd6a2 100644 --- a/apps/server/src/device/LocalDeviceHost.ts +++ b/apps/server/src/device/LocalDeviceHost.ts @@ -55,6 +55,11 @@ import { const HUB_READY_TIMEOUT_MS = 30_000; const DAEMON_READY_TIMEOUT_MS = 30_000; const DAEMON_POLL_MS = 100; +const DAEMON_HEALTH_TIMEOUT_MS = 3_000; +const DAEMON_HEALTH_ATTEMPTS = 3; +// agent-device waits up to 10 s after SIGTERM and 2 s after SIGKILL before it +// reports whether the daemon exited. +const DAEMON_RECOVERY_STOP_TIMEOUT = Duration.seconds(15); const HUB_RESTART_STABLE_UPTIME_MS = 60_000; const HUB_RESTART_MAX_DELAY_MS = 30_000; @@ -553,17 +558,35 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { entryPath: agentTool.entryPath, }); if (existing._tag === "Some") { - const alive = yield* HttpClient.withScope(httpClient) + const probe = HttpClient.withScope(httpClient) .get(`http://127.0.0.1:${existing.value.httpPort}/health`) .pipe( - Effect.timeout(Duration.seconds(2)), Effect.flatMap((response) => response.arrayBuffer.pipe(Effect.as(response.status === 200)), ), + Effect.timeout(Duration.millis(DAEMON_HEALTH_TIMEOUT_MS)), Effect.scoped, Effect.orElseSucceed(() => false), ); - if (alive) return toEndpoint(existing.value); + // A daemon busy with a long command can miss one probe; restarting it then + // would race its live lock and wedge every later start. + for (let attempt = 1; attempt <= DAEMON_HEALTH_ATTEMPTS; attempt++) { + if (yield* probe) return toEndpoint(existing.value); + } + // Stop through agent-device while daemon.json still names the process; once + // the file is gone it reports "not-running" and the daemon keeps daemon.lock. + const stopped = yield* runAgentDeviceDaemonStop( + { entryPath: agentTool.entryPath, nodePath }, + DAEMON_RECOVERY_STOP_TIMEOUT, + ).pipe(Effect.result); + if (stopped._tag === "Failure" || stopped.success.timedOut || stopped.success.code !== 0) { + yield* Effect.logWarning( + "Could not stop an unresponsive agent-device daemon", + stopped._tag === "Failure" + ? { cause: stopped.failure } + : { code: stopped.success.code, timedOut: stopped.success.timedOut }, + ); + } yield* fs.remove(daemonFilePath(), { force: true }).pipe(Effect.ignore); } // No `daemon start` exists; `devices` is the cheapest command that starts and awaits it. @@ -626,6 +649,24 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { } }); + const runAgentDeviceDaemonStop = ( + agentTool: { readonly entryPath: string; readonly nodePath: string }, + timeout: Duration.Duration, + ) => + runner.run({ + command: agentTool.nodePath, + args: [ + agentTool.entryPath, + "daemon", + "stop", + "--state-dir", + agentDeviceStateDir(path, config.stateDir), + ], + env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, + timeout, + timeoutBehavior: "timedOutResult", + }); + // The daemon may predate this process (an earlier server was killed, possibly on an // older pin), so stop it with the CLI of its recorded version when that is installed. const stopAgentDeviceDaemon = Effect.gen(function* () { @@ -646,19 +687,7 @@ export const make = Effect.fn("LocalDeviceHost.make")(function* () { const nodePath = yield* resolveNodeExecutable("Local device support", hostEnvironment).pipe( Effect.provideService(HostProcess.Platform, hostPlatform), ); - yield* runner.run({ - command: nodePath, - args: [ - entryPath, - "daemon", - "stop", - "--state-dir", - agentDeviceStateDir(path, config.stateDir), - ], - env: { ...hostEnvironment, AGENT_DEVICE_NO_UPDATE_NOTIFIER: "1" }, - timeout: Duration.seconds(10), - timeoutBehavior: "timedOutResult", - }); + yield* runAgentDeviceDaemonStop({ entryPath, nodePath }, Duration.seconds(10)); }).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path),