From 6e9a460cd4ee1e7986f9c1f08d25479050c8ae68 Mon Sep 17 00:00:00 2001 From: greyxp1 Date: Sat, 31 Oct 2026 16:42:34 -0400 Subject: [PATCH 1/5] fix(connect): preserve managed tunnels across server restarts --- .../src/updates/DesktopUpdates.test.ts | 33 +-- apps/desktop/src/updates/DesktopUpdates.ts | 32 +-- .../desktop/src/updates/updatesTestHarness.ts | 4 - .../src/cloud/CloudLink.lifecycle.test.ts | 241 ++---------------- apps/server/src/cloud/CloudLink.ts | 84 ------ apps/server/src/cloud/serviceProtocol.ts | 2 - apps/server/src/cloud/updateHandoff.ts | 60 ----- apps/server/src/server.ts | 23 -- apps/server/src/serviceLauncher.ts | 12 +- docs/internals/t3-connect.md | 18 +- infra/relay/.env.example | 6 +- infra/relay/src/Config.test.ts | 51 ++++ infra/relay/src/Config.ts | 12 +- .../ManagedEndpointReaper.test.ts | 2 +- .../src/environments/ManagedEndpointReaper.ts | 2 +- packages/contracts/src/desktopBootstrap.ts | 3 - 16 files changed, 105 insertions(+), 480 deletions(-) delete mode 100644 apps/server/src/cloud/updateHandoff.ts create mode 100644 infra/relay/src/Config.test.ts diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index c6bcd46b8c206..6eb0af9cc19f5 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -1,5 +1,4 @@ import { assert, describe, it } from "@effect/vitest"; -import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; @@ -16,7 +15,6 @@ import * as TestClock from "effect/testing/TestClock"; import * as ElectronUpdater from "../electron/ElectronUpdater.ts"; import * as DesktopBackendPool from "../backend/DesktopBackendPool.ts"; import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; -import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; import * as DesktopState from "../app/DesktopState.ts"; import * as DesktopUpdates from "./DesktopUpdates.ts"; import { flushCallbacks, makeHarness } from "./updatesTestHarness.ts"; @@ -389,30 +387,6 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("marks the backend stop for an install as an update restart", () => { - let markersAtStop: ReadonlyArray = []; - const harness = makeHarness({ - stopBackend: Effect.sync(() => { - markersAtStop = [...harness.updateRestartMarkers]; - }), - }); - - return Effect.scoped( - Effect.gen(function* () { - const environment = yield* DesktopEnvironment.DesktopEnvironment; - const updates = yield* DesktopUpdates.DesktopUpdates; - yield* updates.configure; - harness.emit("update-downloaded", { version: "1.2.4" }); - yield* flushCallbacks; - - assert.isTrue((yield* updates.install).accepted); - assert.deepEqual(markersAtStop, [ - environment.path.join(environment.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE), - ]); - }), - ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); - }); - it.effect("records the version an install hands to the installer", () => { let recordsAtStop: ReadonlyArray = []; const harness = makeHarness({ @@ -474,7 +448,7 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); - it.effect("drops the update restart marker when an install is interrupted", () => + it.effect("drops the install record when an install is interrupted", () => Effect.gen(function* () { const stopping = yield* Deferred.make(); const harness = makeHarness({ @@ -490,10 +464,10 @@ describe("DesktopUpdates", () => { const installFiber = yield* updates.install.pipe(Effect.forkScoped); yield* Deferred.await(stopping); - assert.equal(harness.updateRestartMarkers.size, 1); + assert.equal(harness.installRecords.size, 1); yield* Fiber.interrupt(installFiber); - assert.equal(harness.updateRestartMarkers.size, 0); + assert.equal(harness.installRecords.size, 0); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }), @@ -523,7 +497,6 @@ describe("DesktopUpdates", () => { assert.isTrue(result.accepted); assert.isFalse(yield* Ref.get(desktopState.quitting)); assert.deepEqual(harness.installSteps, ["quitAndInstall", "startBackend"]); - assert.equal(harness.updateRestartMarkers.size, 0); assert.equal(harness.installRecords.size, 0); }), ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index 3a0b41cd9e369..a4aef6e0594e4 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -1,5 +1,4 @@ import { - DESKTOP_UPDATE_RESTART_MARKER_FILE, DesktopUpdateChannelSchema, type DesktopRuntimeInfo, type DesktopUpdateActionResult, @@ -608,32 +607,18 @@ export const make = Effect.gen(function* () { ); }).pipe(Effect.withSpan("desktop.updates.downloadAvailableUpdate")); - const updateRestartMarkerDir = environment.path.join(environment.baseDir, "runtime"); - const updateRestartMarkerPath = environment.path.join( - updateRestartMarkerDir, - DESKTOP_UPDATE_RESTART_MARKER_FILE, - ); - const writeUpdateRestartMarker = fileSystem - .makeDirectory(updateRestartMarkerDir, { recursive: true }) - .pipe( - Effect.andThen(fileSystem.writeFileString(updateRestartMarkerPath, "")), - Effect.catch((error) => - logUpdaterWarning("Could not write the update restart marker.", { errorTag: error._tag }), - ), - ); - - // Failed or interrupted installs leave no replacement backend; later quits must release the tunnel. - const removeUpdateRestartMarker = fileSystem - .remove(updateRestartMarkerPath, { force: true }) - .pipe(Effect.ignore); - + const updateRuntimeDir = environment.path.join(environment.baseDir, "runtime"); const installRecordPath = environment.path.join( - updateRestartMarkerDir, + updateRuntimeDir, DESKTOP_UPDATE_INSTALL_RECORD_FILE, ); const writeInstallRecord = (targetVersion: string) => encodeInstallRecord({ fromVersion: environment.appVersion, targetVersion }).pipe( - Effect.flatMap((text) => fileSystem.writeFileString(installRecordPath, text)), + Effect.flatMap((text) => + fileSystem + .makeDirectory(updateRuntimeDir, { recursive: true }) + .pipe(Effect.andThen(fileSystem.writeFileString(installRecordPath, text))), + ), Effect.catch((error) => logUpdaterWarning("Could not record the update install.", { errorTag: error._tag }), ), @@ -671,7 +656,6 @@ export const make = Effect.gen(function* () { finishUpdateAction("install"), Ref.set(desktopState.quitting, false), Ref.set(installRunningBackendsRef, []), - removeUpdateRestartMarker, removeInstallRecord, ], { discard: true }, @@ -688,7 +672,6 @@ export const make = Effect.gen(function* () { if (!ownsRecovery) return; yield* Ref.set(desktopState.quitting, false); - yield* removeUpdateRestartMarker; yield* removeInstallRecord; yield* Effect.gen(function* () { const runningInstances = yield* Ref.getAndSet(installRunningBackendsRef, []); @@ -763,7 +746,6 @@ export const make = Effect.gen(function* () { yield* Ref.set(installRunningBackendsRef, []); return yield* Effect.gen(function* () { - yield* writeUpdateRestartMarker; const { downloadedVersion } = yield* Ref.get(updateStateRef); if (downloadedVersion !== null) yield* writeInstallRecord(downloadedVersion); // Stop pooled backends before quitAndInstall can outrun their scope finalizers. diff --git a/apps/desktop/src/updates/updatesTestHarness.ts b/apps/desktop/src/updates/updatesTestHarness.ts index d8d16b7dbfd62..b1d726f89f178 100644 --- a/apps/desktop/src/updates/updatesTestHarness.ts +++ b/apps/desktop/src/updates/updatesTestHarness.ts @@ -233,7 +233,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { } satisfies DesktopAppSettings.DesktopAppSettings["Service"]) : DesktopAppSettings.layer; - const updateRestartMarkers = new Set(); const installRecords = new Map(); let previousInstallRecord = options.installRecord; const isInstallRecord = (path: string) => @@ -259,11 +258,9 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { writeFileString: (path, data) => Effect.sync(() => { if (isInstallRecord(path)) installRecords.set(path, data); - else updateRestartMarkers.add(path); }), remove: (path) => Effect.sync(() => { - updateRestartMarkers.delete(path); installRecords.delete(path); if (isInstallRecord(path)) previousInstallRecord = undefined; }), @@ -293,7 +290,6 @@ export function makeHarness(options: UpdatesHarnessOptions = {}) { checkCount: () => checkCount, quitAndInstalls: () => quitAndInstallCount, installSteps, - updateRestartMarkers, installRecords, previousInstallRecord: () => previousInstallRecord, downloadCount: () => downloadCount, diff --git a/apps/server/src/cloud/CloudLink.lifecycle.test.ts b/apps/server/src/cloud/CloudLink.lifecycle.test.ts index facd9f8912e53..00ac6b64cd066 100644 --- a/apps/server/src/cloud/CloudLink.lifecycle.test.ts +++ b/apps/server/src/cloud/CloudLink.lifecycle.test.ts @@ -1,10 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -14,14 +11,6 @@ import * as NetAddress from "effect/net/NetAddress"; import { EnvironmentId } from "@t3tools/contracts"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import * as ServerConfigModule from "../config.ts"; -import { writeServiceState } from "../serviceLauncher.ts"; -import { - SERVICE_LAUNCHER_PROTOCOL, - SERVICE_STATE_FILE, - SERVICE_STOP_MARKER_FILE, - type ServiceUpdateRecord, -} from "./serviceProtocol.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; @@ -40,7 +29,6 @@ import { import * as CloudLink from "./CloudLink.ts"; import * as ConnectReachability from "./ConnectReachability.ts"; import { consumeCloudReplayGuards } from "./CloudLink.ts"; -import { pendingServiceUpdateExists } from "./updateHandoff.ts"; import { managedTunnelStartupAction } from "./managedTunnelStartup.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; @@ -99,7 +87,7 @@ describe("consumeCloudReplayGuards", () => { ); }); -describe("releaseManagedTunnelOnShutdown", () => { +describe("managed tunnel recovery", () => { const cliToken: CliTokenManager.PersistedToken = { accessToken: "cli-access-token", refreshToken: "cli-refresh-token", @@ -129,7 +117,7 @@ describe("releaseManagedTunnelOnShutdown", () => { return { store, values }; } - interface ReleaseHarness { + interface RecoveryHarness { readonly store: ServerSecretStore.ServerSecretStore["Service"]; readonly applyConfigCalls: Array; readonly requests: Array; @@ -140,22 +128,8 @@ describe("releaseManagedTunnelOnShutdown", () => { readonly getExisting?: CliTokenManager.CloudCliTokenManager["Service"]["getExisting"]; } - const writeLauncherState = (update: ServiceUpdateRecord) => - Effect.gen(function* () { - const path = yield* Path.Path; - const config = yield* ServerConfigModule.ServerConfig; - const statePath = path.join(config.baseDir, "runtime", SERVICE_STATE_FILE); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "0.0.30", - update, - }), - ); - }); - - const provideReleaseHarness = - (harness: ReleaseHarness) => + const provideRecoveryHarness = + (harness: RecoveryHarness) => (effect: Effect.Effect) => effect.pipe( Effect.provide(CloudLink.layer), @@ -233,11 +207,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ), ), - Effect.provide( - ServerConfigModule.layerTest("/", { prefix: "t3-http-release-test-" }).pipe( - Layer.provideMerge(NodeServices.layer), - ), - ), + Effect.provide(NodeServices.layer), Effect.scoped, ); @@ -261,172 +231,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.reconcileDesiredLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull(); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(applyConfigCalls).toEqual([null]); - expect(requests).toHaveLength(1); - const request = requests[0]!; - expect(request.method).toBe("DELETE"); - expect(request.url).toBe( - "https://relay.example.test/v1/client/environment-links/env_123/tunnel", - ); - expect(request.headers.authorization).toBe("Bearer cli-access-token"); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("leaves the tunnel of a web/mobile-installed link untouched", () => { - const { store, values } = makeMemorySecretStore([ - [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], - [RELAY_URL_SECRET, "https://relay.example.test"], - ]); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(applyConfigCalls).toEqual([]); - expect(requests).toEqual([]); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("keeps the tunnel when shutdown hands off to a pending update", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - dbPath: "/tmp/state.sqlite", - status: "pending", - }); - - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(applyConfigCalls).toEqual([]); - expect(requests).toEqual([]); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("still releases a pending update when the launcher is stopping", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - dbPath: "/tmp/state.sqlite", - status: "pending", - }); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const config = yield* ServerConfigModule.ServerConfig; - yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), ""); - - expect(yield* pendingServiceUpdateExists).toBe(true); - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - // regression: 808d6853555 keep the managed tunnel across remote update restarts - it.effect("still releases when the recorded update already settled", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - yield* writeLauncherState({ - id: "update-1", - fromVersion: "0.0.30", - targetVersion: "0.0.31", - status: "committed", - }); - - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); - }); - - it.effect("keeps a runtime config that a fast restart replaced mid-release", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - const freshConfig = new TextEncoder().encode("fresh-runtime-config"); - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(true); - - expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); - }).pipe( - provideReleaseHarness({ - store, - applyConfigCalls, - requests, - respond: () => { - values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); - return Response.json({ ok: true }); - }, - }), - ); - }); - - it.effect("keeps the stored connector token when the relay skipped the release", () => { - const { store, values } = makeMemorySecretStore(managedLinkSecrets); - const applyConfigCalls: Array = []; - const requests: Array = []; - - return Effect.gen(function* () { - const link = yield* CloudLink.CloudLink; - const released = yield* link.releaseManagedTunnelOnShutdown(); - - expect(released).toBe(false); - expect(requests).toHaveLength(1); - expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); - }).pipe( - provideReleaseHarness({ - store, - applyConfigCalls, - requests, - respond: () => Response.json({ ok: false }), - }), - ); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("registers an existing tunnel and starts the confirmed connector", () => { @@ -466,7 +271,7 @@ describe("releaseManagedTunnelOnShutdown", () => { } expect(applyConfigCalls).toHaveLength(1); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -507,7 +312,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(applyConfigCalls).toEqual([null, config]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -543,7 +348,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(values.has(RELAY_ENVIRONMENT_CREDENTIAL_SECRET)).toBe(true); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(true); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -579,7 +384,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls: [], requests: [], @@ -623,7 +428,7 @@ describe("releaseManagedTunnelOnShutdown", () => { endpointRuntime: null, }); expect(yield* link.linkState()).toMatchObject({ linked: true, linkRevoked: false }); - }).pipe(provideReleaseHarness({ store, applyConfigCalls: [], requests: [] })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls: [], requests: [] })); }); it.effect( @@ -651,7 +456,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.startManagedTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true); expect(applyConfigCalls).toEqual([config]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }, ); @@ -680,7 +485,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.startManagedTunnelIfOriginConfirmed(origin)).toBe(false); expect(applyConfigCalls).toEqual([]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); // regression: #9386 remove tunnels after hosts go offline @@ -707,7 +512,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toBe(true); expect(applyConfigCalls).toEqual([config]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }, ); @@ -733,7 +538,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(applyConfigCalls).toEqual([]); expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -786,7 +591,7 @@ describe("releaseManagedTunnelOnShutdown", () => { }); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("recovers a web-linked tunnel and clears its revoked mark", () => { @@ -822,7 +627,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toEqual(nextConfig); expect(values.has(CLOUD_LINK_REVOKED)).toBe(false); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -852,7 +657,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(yield* link.recoverManagedTunnel("http://127.0.0.1:3773")).toBe(false); expect(applyConfigCalls).toEqual([]); expect(requests).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { @@ -879,7 +684,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ).toBe(false); expect(requests).toEqual([]); expect(applyConfigCalls).toEqual([]); - }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store, applyConfigCalls, requests })); }); it.effect.each([ @@ -914,7 +719,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(requests).toHaveLength(1); expect(applyConfigCalls).toEqual([]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -941,7 +746,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(error.message).toContain("Trace ID: trace-expired"); expect(CloudLink.shouldRetryCloudLink(error)).toBe(true); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -975,7 +780,7 @@ describe("releaseManagedTunnelOnShutdown", () => { expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); expect(applyConfigCalls).toEqual([]); }).pipe( - provideReleaseHarness({ + provideRecoveryHarness({ store, applyConfigCalls, requests, @@ -1046,6 +851,6 @@ describe("releaseManagedTunnelOnShutdown", () => { managedTunnelActive: false, }); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); - }).pipe(provideReleaseHarness({ store: failingStore, applyConfigCalls, requests })); + }).pipe(provideRecoveryHarness({ store: failingStore, applyConfigCalls, requests })); }); }); diff --git a/apps/server/src/cloud/CloudLink.ts b/apps/server/src/cloud/CloudLink.ts index 99530612463e1..5fc04e50771f0 100644 --- a/apps/server/src/cloud/CloudLink.ts +++ b/apps/server/src/cloud/CloudLink.ts @@ -27,7 +27,6 @@ import { RelayManagedEndpointRecoveryRegistrationResponse, RelayManagedEndpointRecoveryResponse, type RelayManagedEndpointRuntimeConfig, - RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { @@ -47,10 +46,8 @@ import * as Crypto from "effect/Crypto"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Path from "effect/Path"; import type * as PlatformError from "effect/PlatformError"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; @@ -61,12 +58,10 @@ import { HttpClientResponse, type HttpServerRequest, } from "effect/http"; -import type * as HttpClientError from "effect/http/HttpClientError"; import * as HttpServer from "effect/http/HttpServer"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts"; @@ -116,7 +111,6 @@ import { parseManagedEndpointLocalOrigin, requestAbsoluteUrl, } from "./linkChecks.ts"; -import { desktopUpdateRestartPending, pendingUpdateHandoffExists } from "./updateHandoff.ts"; const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; @@ -506,12 +500,6 @@ export class CloudLink extends Context.Service< | CloudLinkOriginInvalidError | ServerSecretStore.SecretStoreError >; - readonly releaseManagedTunnelOnShutdown: () => Effect.Effect< - boolean, - | CloudLinkBackgroundError - | CliTokenManager.CloudCliTokenManagerError - | HttpClientError.HttpClientError - >; } >()("t3/cloud/CloudLink") {} @@ -526,19 +514,7 @@ const make = Effect.gen(function* () { const httpServer = yield* HttpServer.HttpServer; const awarenessRelay = yield* AgentAwarenessRelay.AgentAwarenessRelay; const crypto = yield* Crypto.Crypto; - const config = yield* ServerConfig.ServerConfig; - const fileSystem = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const withSecrets = Effect.provideService(ServerSecretStore.ServerSecretStore, secrets); - const withRuntimeFiles = ( - effect: Effect.Effect, - ) => - effect.pipe( - Effect.provideService(ServerConfig.ServerConfig, config), - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - ); const validateLinkedCloudUser = ( cloudUserId: string, @@ -1203,65 +1179,6 @@ const make = Effect.gen(function* () { }), ); - // Cloudflare bills provisioned tunnels even while environments are offline. - const releaseManagedTunnelOnShutdown = Effect.fn( - "environment.cloud.releaseManagedTunnelOnShutdown", - )(function* () { - const runtimeConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if (Option.isNone(runtimeConfig)) { - return false; - } - // CLI authorization releases only CLI-desired managed links; web/mobile links recover by credential and expire through the relay reaper. - if ( - !(yield* readCliDesiredCloudLink.pipe(withSecrets)) || - (yield* readCliDesiredLinkMode.pipe(withSecrets)) !== "managed" - ) { - return false; - } - // Update handoffs reuse the tunnel: replacing it adds 1-2 minutes of hostname propagation. - if ( - (yield* withRuntimeFiles(desktopUpdateRestartPending)) || - (yield* withRuntimeFiles(pendingUpdateHandoffExists)) - ) { - yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); - return false; - } - const token = yield* cliTokenManager.getExisting; - if (Option.isNone(token)) { - return false; - } - // Use the persisted relay URL; T3CODE_RELAY_URL may have changed since linking. - const relayUrl = yield* secrets.get(RELAY_URL_SECRET); - if (Option.isNone(relayUrl)) { - return false; - } - const environmentId = yield* environment.getEnvironmentId; - yield* endpointRuntime.applyConfig(null); - const response = yield* HttpClientRequest.delete( - `${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`, - ).pipe( - HttpClientRequest.bearerToken(token.value.accessToken), - httpClient.execute, - Effect.flatMap(filterRelayResponse), - Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)), - withRelayClientTracing, - ); - // ok:false means concurrent provisioning owns the tunnel; preserve its config. - if (!response.ok) { - return false; - } - // Released tokens are dead; remove only this shutdown's config to preserve a fast restart's replacement. - const storedConfig = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if ( - Option.isSome(storedConfig) && - bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value) - ) { - yield* secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - yield* secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); - } - return true; - }); - const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* () { const [ cloudUserId, @@ -1630,7 +1547,6 @@ const make = Effect.gen(function* () { registerManagedTunnelRecovery, recoverManagedTunnel, startManagedTunnelIfOriginConfirmed, - releaseManagedTunnelOnShutdown, }); }); diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 3b8ea548d93da..670c9ca325d9b 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -5,8 +5,6 @@ export const SERVICE_LAUNCHER_PROTOCOL = 3 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_ENV_FILE = "service.env"; export const SERVICE_STATE_FILE = "service-state.json"; -/** Written before explicit stop to distinguish shutdown from update handoff. */ -export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; diff --git a/apps/server/src/cloud/updateHandoff.ts b/apps/server/src/cloud/updateHandoff.ts deleted file mode 100644 index 27b66c7f5b2ee..0000000000000 --- a/apps/server/src/cloud/updateHandoff.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts"; -import * as Clock from "effect/Clock"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Option from "effect/Option"; -import * as Path from "effect/Path"; - -import * as ServerConfig from "../config.ts"; -import { - SERVICE_STATE_FILE, - SERVICE_STOP_MARKER_FILE, - serviceStateHasPendingUpdate, -} from "./serviceProtocol.ts"; - -const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1); - -export const pendingServiceUpdateExists = Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const runtimeDir = path.join(config.baseDir, "runtime"); - const stateText = yield* fs - .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE)) - .pipe(Effect.option); - return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); -}); - -export const pendingUpdateHandoffExists = Effect.gen(function* () { - if (!(yield* pendingServiceUpdateExists)) { - return false; - } - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const runtimeDir = path.join(config.baseDir, "runtime"); - const stopping = yield* fs - .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE)) - .pipe(Effect.orElseSucceed(() => false)); - return !stopping; -}); - -// Reading consumes it, so shutdown checks it first. -export const desktopUpdateRestartPending = Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const markerPath = path.join(config.baseDir, "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE); - const marker = yield* fs.stat(markerPath).pipe(Effect.option); - if (Option.isNone(marker)) { - return false; - } - yield* fs.remove(markerPath).pipe(Effect.ignore); - const now = yield* Clock.currentTimeMillis; - return Option.match(marker.value.mtime, { - onNone: () => false, - onSome: (writtenAt) => - now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL), - }); -}); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 452addfb4a667..eceeb40dcfc01 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -153,7 +153,6 @@ import * as CloudHttp from "./cloud/http.ts"; import * as GuestHttp from "./guest/http.ts"; import * as CloudLink from "./cloud/CloudLink.ts"; import * as ConnectReachability from "./cloud/ConnectReachability.ts"; -import { pendingServiceUpdateExists } from "./cloud/updateHandoff.ts"; import * as RelayTracing from "./cloud/relayTracing.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; import { @@ -882,30 +881,8 @@ const layerMakeServer = Layer.unwrap( const layerCloudDesiredLinkReconcile = Layer.effectDiscard( Effect.gen(function* () { const cloudLink = yield* CloudLink.CloudLink; - const releaseManagedTunnel = cloudLink.releaseManagedTunnelOnShutdown().pipe( - Effect.timeout("10 seconds"), - Effect.tap((released) => - released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, - ), - Effect.catchCause((cause) => - Effect.logWarning( - "Failed to release the managed tunnel on shutdown; the next link reuses it", - { errors: Cause.prettyErrors(cause).map((error) => error.message) }, - ), - ), - Effect.asVoid, - ); - // Trials own cleanup immediately after the old server exits, retaining tunnels for commit/rollback unless explicitly stopped. - // Other runtimes wait for activation so failed standbys cannot tear down active tunnels. - const cleanupBeforeActivation = yield* pendingServiceUpdateExists; - if (cleanupBeforeActivation) { - yield* Effect.addFinalizer(() => releaseManagedTunnel); - } yield* ServerActivation.forkParked( Effect.gen(function* () { - if (!cleanupBeforeActivation) { - yield* Effect.addFinalizer(() => releaseManagedTunnel); - } const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 9f764b788efe2..da27e8fbf879a 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -3,7 +3,7 @@ // The launcher must survive unloadable server versions; use Node built-ins without Effect. import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; -import * as NodeFS from "node:fs"; +import type * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeUtil from "node:util"; @@ -26,7 +26,6 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, SERVICE_RESTART_PENDING_FILE, - SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; @@ -429,8 +428,6 @@ async function terminateChild( } } -const stopMarkerPath = (baseDir: string) => - NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); const restartPendingPath = (baseDir: string) => NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); @@ -497,10 +494,6 @@ export class Launcher { } async stop(signal: NodeJS.Signals): Promise { - // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer. - try { - NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); - } catch {} if (this.#stopRequested || this.#stopping) { await this.#completion.promise.catch(() => undefined); return; @@ -525,8 +518,7 @@ export class Launcher { } async #recover(): Promise { - // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. - await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); + // Clear a deferred restart only after the requested version starts. const restartPending = restartPendingPath(this.#baseDir); const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch(() => undefined); if (awaitedVersion?.trim() === this.#state.activeVersion) { diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 4c9bbcb780bee..825ee399bef83 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -71,17 +71,11 @@ reconciles that intent. CLI logout removes the stored cloud credential and disables exposure without uninstalling the environment's background service. Managed allocations belong to a user/environment pair. Provisioning checkpoints -external tunnel and DNS resources so retries can reconcile partial work. A -normal shutdown of a CLI-managed link releases its tunnel to avoid paying for -an idle resource, retaining the hostname reservation for the next startup. -It also retains the allocation record so the environment remains "offline" -rather than becoming "not authorized". - -Two cases must retain the tunnel across shutdown. A link installed through a -client has no startup provisioning path and depends on its stored connector -token. An update handoff immediately starts a replacement server, and replacing -the tunnel would add routing propagation delay to every update. These exceptions -belong to [shutdown handling](../../apps/server/src/cloud/CloudLink.ts). +external tunnel and DNS resources so retries can reconcile partial work. +Server shutdown stops the connector but keeps the tunnel and its stored config, +so restarts reconnect without provisioning or routing propagation. Explicit +unlink removes the link and tunnel. The relay reclaims tunnels that remain +offline after the grace period below. Release and unlink claim the allocation generation before deleting external resources. A delayed cleanup must not delete a tunnel reused by a concurrent @@ -95,7 +89,7 @@ teardown retains enough state to retry. See the Cloudflare bills a tunnel whether or not a connector is attached, so a laptop that sleeps with a linked environment leaves a paid tunnel behind. The relay's five-minute maintenance job can reclaim those tunnels. `RELAY_TUNNEL_CLEANUP_MODE` -selects `off`, `dry-run`, or `enabled`, with `off` as the default. The mode is +selects `off`, `dry-run`, or `enabled`, with `enabled` as the default. The mode is read at deploy time, so changing it means a relay deploy, not a variable flip. A candidate is a same-stage tunnel that Cloudflare reports down for at least five minutes, or one that never connected and is at least an hour old. The diff --git a/infra/relay/.env.example b/infra/relay/.env.example index 87a9a1ca32cb8..639515f6121f7 100644 --- a/infra/relay/.env.example +++ b/infra/relay/.env.example @@ -5,9 +5,9 @@ RELAY_API_ZONE_NAME=example.com RELAY_TUNNEL_ZONE_NAME=tunnels.example.com -# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup -# logs, then set enabled. Unset and off both disable cleanup. -# RELAY_TUNNEL_CLEANUP_MODE=off +# Optional: inactive tunnel cleanup. Defaults to enabled for recoverable tunnels. +# Use dry-run to inspect candidates or off to disable cleanup. +# RELAY_TUNNEL_CLEANUP_MODE=enabled # Optional: cleanup for tunnels whose host never registered recovery, once they # have been down for 30 days. Same values, independent of the mode above. # RELAY_LEGACY_TUNNEL_CLEANUP_MODE=off diff --git a/infra/relay/src/Config.test.ts b/infra/relay/src/Config.test.ts new file mode 100644 index 0000000000000..c458838dd347d --- /dev/null +++ b/infra/relay/src/Config.test.ts @@ -0,0 +1,51 @@ +import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; + +import { + legacyManagedEndpointCleanupModeConfig, + managedEndpointCleanupModeConfig, +} from "./Config.ts"; + +it.effect.each([ + { name: "missing", env: {}, expected: "enabled" }, + { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "enabled" }, + { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "enabled" }, + { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" }, + { + name: "dry-run", + env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" }, + expected: "dry-run", + }, + { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" }, +] as const)("loads $name cleanup mode as $expected", ({ env, expected }) => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ env }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected); + }), +); + +it.effect("rejects an invalid cleanup mode", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" }, + }); + const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider)); + + expect(error._tag).toBe("ConfigError"); + expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"'); + }), +); + +it.effect("reads the legacy cleanup mode independently of the main one", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled", RELAY_LEGACY_TUNNEL_CLEANUP_MODE: "dry-run" }, + }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe("enabled"); + expect(yield* legacyManagedEndpointCleanupModeConfig.parse(provider)).toBe("dry-run"); + expect( + yield* legacyManagedEndpointCleanupModeConfig.parse(ConfigProvider.fromEnv({ env: {} })), + ).toBe("off"); + }), +); diff --git a/infra/relay/src/Config.ts b/infra/relay/src/Config.ts index ecc71a5a96a47..00af2b5616094 100644 --- a/infra/relay/src/Config.ts +++ b/infra/relay/src/Config.ts @@ -15,10 +15,10 @@ const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpo const RELAY_TUNNEL_CLEANUP_MODE = "RELAY_TUNNEL_CLEANUP_MODE"; const RELAY_LEGACY_TUNNEL_CLEANUP_MODE = "RELAY_LEGACY_TUNNEL_CLEANUP_MODE"; -const cleanupModeConfig = (name: string) => +const cleanupModeConfig = (name: string, defaultMode: ManagedEndpointCleanupMode) => Config.String(name).pipe( - Config.withDefault("off"), - Config.map((value) => value.trim() || "off"), + Config.withDefault(defaultMode), + Config.map((value) => value.trim() || defaultMode), Config.mapEffect((value) => decodeManagedEndpointCleanupMode(value).pipe( Effect.mapError((error) => new Config.ConfigError(error)), @@ -26,9 +26,13 @@ const cleanupModeConfig = (name: string) => ), ); -export const managedEndpointCleanupModeConfig = cleanupModeConfig(RELAY_TUNNEL_CLEANUP_MODE); +export const managedEndpointCleanupModeConfig = cleanupModeConfig( + RELAY_TUNNEL_CLEANUP_MODE, + "enabled", +); export const legacyManagedEndpointCleanupModeConfig = cleanupModeConfig( RELAY_LEGACY_TUNNEL_CLEANUP_MODE, + "off", ); const RELAY_LEGACY_TUNNEL_GRACE_MINUTES = "RELAY_LEGACY_TUNNEL_GRACE_MINUTES"; diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index d2a9bb7f6c3b0..c3a33cfe0171f 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -236,7 +236,7 @@ function harness(input?: { cloudMintPublicKey: "cloud-public-key", managedEndpointBaseDomain: "example.test", managedEndpointNamespace: input?.namespace ?? "prod", - managedEndpointCleanupMode: input?.cleanupMode ?? "enabled", + ...(input?.cleanupMode === undefined ? {} : { managedEndpointCleanupMode: input.cleanupMode }), legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off", ...(input?.legacyTunnelGraceMinutes === undefined ? {} diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index f800dd8c070d0..48f2502bdeb4b 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -154,7 +154,7 @@ const make = Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; const sweep = Effect.gen(function* () { - const mode = config.managedEndpointCleanupMode ?? "off"; + const mode = config.managedEndpointCleanupMode ?? "enabled"; const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off"; const namespace = config.managedEndpointNamespace; if ((mode === "off" && legacyMode === "off") || !namespace) { diff --git a/packages/contracts/src/desktopBootstrap.ts b/packages/contracts/src/desktopBootstrap.ts index a77e14b555420..5af3928039afd 100644 --- a/packages/contracts/src/desktopBootstrap.ts +++ b/packages/contracts/src/desktopBootstrap.ts @@ -30,6 +30,3 @@ export const DesktopBackendBootstrap = Schema.Struct({ }); export type DesktopBackendBootstrap = typeof DesktopBackendBootstrap.Type; - -/** Written to `/runtime` before update shutdown so the backend retains its managed tunnel. */ -export const DESKTOP_UPDATE_RESTART_MARKER_FILE = "desktop-update-restart"; From 111bdba26d7271e58d4fa6c842e015c297d36f40 Mon Sep 17 00:00:00 2001 From: greyxp1 Date: Sat, 31 Oct 2026 17:19:47 -0400 Subject: [PATCH 2/5] test(connect): cover tunnel retention across server restarts --- apps/server/src/server.lifecycle.test.ts | 207 +++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 apps/server/src/server.lifecycle.test.ts diff --git a/apps/server/src/server.lifecycle.test.ts b/apps/server/src/server.lifecycle.test.ts new file mode 100644 index 0000000000000..2843f8a144dd5 --- /dev/null +++ b/apps/server/src/server.lifecycle.test.ts @@ -0,0 +1,207 @@ +// @effect-diagnostics nodeBuiltinImport:off -- Exercises CLI subprocess shutdown against a local relay fixture. +import * as NodeChildProcess from "node:child_process"; +import * as NodeEvents from "node:events"; +import * as NodeHttp from "node:http"; +import * as NodeURL from "node:url"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as NetService from "@t3tools/shared/Net"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; + +import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeConfirmedOrigin, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_ISSUER_SECRET, + RELAY_URL_SECRET, +} from "./cloud/config.ts"; +import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./cloud/CliState.ts"; +import { makeProcessProbe, processProbeSource } from "./testUtils/processProbe.ts"; + +it.live("retains a managed tunnel on server shutdown and reuses it after restart", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const net = yield* NetService.NetService; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-tunnel-lifecycle-" }); + const secretsDir = path.join(root, "userdata", "secrets"); + yield* fs.makeDirectory(secretsDir, { recursive: true }); + const requests: string[] = []; + const relay = yield* Effect.acquireRelease( + Effect.promise(async () => { + const server = NodeHttp.createServer((request, response) => { + requests.push(`${request.method} ${request.url}`); + request.resume(); + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify(request.method === "DELETE" ? { ok: true } : { status: "ready" }), + ); + }); + server.listen(0, "127.0.0.1"); + await NodeEvents.EventEmitter.once(server, "listening"); + return server; + }), + (server) => + Effect.promise(() => new Promise((resolve) => server.close(() => resolve()))), + ); + const address = relay.address(); + if (address === null || typeof address === "string") return yield* Effect.die("No relay port"); + const relayUrl = `http://127.0.0.1:${address.port}`; + const port = yield* net.reserveLoopbackPort(); + const config = { + providerKind: "cloudflare_tunnel", + tunnelId: "existing-tunnel", + connectorToken: "existing-token", + }; + const configJson = JSON.stringify(config); + const confirmedOrigin = { + config, + origin: { localHttpHost: "127.0.0.1", localHttpPort: port }, + }; + yield* Effect.forEach( + Object.entries({ + [CLOUD_ENDPOINT_RUNTIME_CONFIG]: configJson, + [CLOUD_ENDPOINT_CONFIRMED_ORIGIN]: JSON.stringify(confirmedOrigin), + [CLOUD_CLI_DESIRED_LINK_SECRET]: "managed", + [CLOUD_LINKED_USER_ID]: "user-123", + [RELAY_URL_SECRET]: relayUrl, + [RELAY_ISSUER_SECRET]: "https://relay.example.test", + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET]: "environment-credential", + "cloud-cli-oauth-token": JSON.stringify({ + accessToken: "cli-access-token", + refreshToken: "cli-refresh-token", + expiresAtEpochMs: Number.MAX_SAFE_INTEGER, + }), + }), + ([name, value]) => fs.writeFileString(path.join(secretsDir, `${name}.bin`), value), + ); + yield* fs.writeFileString(path.join(root, "userdata", "environment-id"), "env_123"); + const bin = yield* path.fromFileUrl(new URL("./bin.ts", import.meta.url)); + const inheritedEnv = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !/^(?:T3CODE_|VITE_)/u.test(name)), + ); + + const startAndStop = Effect.fn("test.managedTunnel.startAndStop")( + function* () { + const connector = yield* makeProcessProbe; + const preload = path.join(root, `connector-${connector.port}.mjs`); + // Node acts as cloudflared so the fixture also works without a shell launcher on Windows. + yield* fs.writeFileString( + preload, + ` + import { basename } from "node:path"; + const command = basename(process.argv[1] ?? ""); + if (command === "version") { + console.log("cloudflared version 2026.10.4"); + process.exit(0); + } + if (command === "tunnel") { + ${processProbeSource(connector.port)} + console.error("INF Registered tunnel connection connIndex=0"); + await new Promise((resolve) => probe.on("close", resolve)); + process.exit(0); + } + process.on("message", (message) => { + if (message === "shutdown") process.emit("SIGTERM"); + }); + `, + ); + const child = yield* Effect.acquireRelease( + Effect.sync(() => + NodeChildProcess.fork( + bin, + ["serve", "--base-dir", root, "--host", "127.0.0.1", "--port", String(port)], + { + cwd: root, + execArgv: [], + silent: true, + env: { + ...inheritedEnv, + NODE_OPTIONS: `--import=${NodeURL.pathToFileURL(preload).href}`, + T3CODE_CLOUDFLARED_PATH: process.execPath, + T3CODE_RELAY_URL: "https://relay.example.test", + T3CODE_CLERK_PUBLISHABLE_KEY: `pk_test_${Buffer.from("clerk.example.test$").toString("base64")}`, + T3CODE_CLERK_CLI_OAUTH_CLIENT_ID: "test-client", + T3CODE_TELEMETRY_ENABLED: "false", + }, + }, + ), + ), + (process) => + Effect.promise(async () => { + if ( + process.pid === undefined || + process.exitCode !== null || + process.signalCode !== null + ) + return; + const exited = NodeEvents.EventEmitter.once(process, "close"); + process.kill("SIGKILL"); + await exited; + }), + ); + const ready = Promise.withResolvers(); + const exited = NodeEvents.EventEmitter.once(child, "close"); + let output = ""; + for (const stream of [child.stdout, child.stderr]) { + stream?.on("data", (chunk: Buffer) => { + output += chunk.toString(); + if ( + output.includes("T3 Code server is ready.") && + output.includes("T3 Connect managed tunnel recovery registered") && + output.includes("Relay client tunnel connection registered") + ) { + ready.resolve(); + } + }); + } + yield* Effect.promise(() => + Promise.race([ + ready.promise, + exited.then(() => { + throw new Error(`Server exited before tunnel registration:\n${output}`); + }), + ]), + ); + yield* connector.started; + yield* Effect.promise(async () => { + child.send("shutdown"); + await exited; + }); + yield* connector.exited; + expect(output).toContain("V2 orchestration shutdown reconciliation completed"); + }, + Effect.scoped, + // The shared process probe uses TestClock.withLive for its event deadlines. + Effect.provide(TestClock.layer()), + ); + + yield* startAndStop(); + expect(requests).toEqual(["POST /v1/environments/env_123/tunnel/recovery"]); + expect( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_RUNTIME_CONFIG}.bin`)), + ).toBe(configJson); + expect( + decodeConfirmedOrigin( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_CONFIRMED_ORIGIN}.bin`)), + ), + ).toEqual(Option.some(confirmedOrigin)); + + yield* startAndStop(); + expect(requests).toEqual([ + "POST /v1/environments/env_123/tunnel/recovery", + "POST /v1/environments/env_123/tunnel/recovery", + ]); + expect( + yield* fs.readFileString(path.join(secretsDir, `${CLOUD_ENDPOINT_RUNTIME_CONFIG}.bin`)), + ).toBe(configJson); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NetService.layer))), +); From 4bd2400e98c6fde9d52f34b873f2875277ec9616 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 12 Nov 2026 00:31:37 +0000 Subject: [PATCH 3/5] test(connect): keep launcher stop coverage focused --- apps/server/src/serviceLauncher.test.ts | 112 ++++++++++++------------ 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 7cd777e7afb8f..d0f7c0635a4af 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1119,24 +1119,22 @@ if (context.update?.status === "pending") { }), ); - it.effect( - "holds its lock while it serves and stops the server over IPC without a tunnel-release marker", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - const reportPath = path.join(root, "stopped-with-marker"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "database"); - const encoded = JSON.stringify({ - databasePath, - reportPath, - markerPath: path.join(root, "runtime", ".service-stopping"), - }); - const childSource = ` + it.effect("holds its lock while it serves and stops the server over IPC, no marker", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const reportPath = path.join(root, "stopped-with-marker"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + const encoded = JSON.stringify({ + databasePath, + reportPath, + markerPath: path.join(root, "runtime", ".service-stopping"), + }); + const childSource = ` const fs = require("node:fs"); const paths = ${encoded}; process.on("message", (message) => { @@ -1146,48 +1144,48 @@ process.on("message", (message) => { }); process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }); `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, - ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, ); - const lockPath = serviceLauncherLockPath(root); - const serving = Promise.withResolvers(); - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { - lockPath, - stopChildOverIpc: true, - // The update request proves the child is up; refusing it leaves the server running. - backupSpaceProbe: { - readFreeBytes: () => { - serving.resolve(); - return Promise.resolve(0); - }, - canClone: () => Promise.resolve(false), + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const lockPath = serviceLauncherLockPath(root); + const serving = Promise.withResolvers(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + lockPath, + stopChildOverIpc: true, + // The update request proves the child is up; refusing it leaves the server running. + backupSpaceProbe: { + readFreeBytes: () => { + serving.resolve(); + return Promise.resolve(0); }, + canClone: () => Promise.resolve(false), }, - ); - const running = launcher.run(); - yield* Effect.promise(() => Promise.race([serving.promise, running])); - - assert.isTrue(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath))); - // What losing the task's console does. - yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); - assert.equal(yield* fs.readFileString(reportPath), "false"); - assert.isFalse(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath))); - }), + }, + ); + const running = launcher.run(); + yield* Effect.promise(() => Promise.race([serving.promise, running])); + + assert.isTrue(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath))); + // What losing the task's console does. + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + assert.equal(yield* fs.readFileString(reportPath), "false"); + assert.isFalse(yield* Effect.promise(() => serviceLauncherIsRunning(lockPath))); + }), ); it.effect("leaves a pending update alone when stopped while it starts up", () => From e3b792503b6abd377d146b7f26ef0935415e33e4 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Thu, 12 Nov 2026 00:36:37 +0000 Subject: [PATCH 4/5] docs(connect): remove stale stop-marker explanation --- apps/server/src/serviceLauncher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 981c91be024fb..02d6af76dcaa8 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -826,7 +826,7 @@ export class Launcher { this.#stopRequested = true; this.#clearTimer(); this.#enqueue(async () => { - // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. + // Let in-progress updates start replacements first. this.#stopping = true; const child = this.#child?.process; this.#child = null; From bbc596ab6b168fae095215edb4522f0151ea3c68 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Sun, 15 Nov 2026 03:15:43 +0000 Subject: [PATCH 5/5] docs(connect): align release cleanup default --- docs/operations/release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index 4e98e516e46be..178c34d48b27a 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -144,7 +144,7 @@ Optional `production` environment variables: - `RELAY_DOMAIN` when overriding the derived `relay.` domain - `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use - `off`. + `enabled`. - `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` with the same values, for tunnels whose host never registered recovery. Missing and blank values use `off`.