diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 68870bd969436..0c016a504c9b4 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -7,9 +7,10 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; import * as NodePath from "node:path"; import * as TestClock from "effect/testing/TestClock"; -import { HttpClient, HttpClientResponse } from "effect/http"; +import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; @@ -189,7 +190,127 @@ it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { validate: () => Effect.die("must not validate a missing archive"), }).pipe(Effect.flip); assert.isTrue(isUnpublishedRelease(yield* failsWith(404))); - assert.isFalse(isUnpublishedRelease(yield* failsWith(500))); + assert.isFalse(isUnpublishedRelease(yield* failsWith(403))); + }), + ); + + it.effect("retries a release download that GitHub answers with a gateway timeout", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-retry-" }); + const checksums = yield* validChecksums; + const requests: string[] = []; + const gatewayTimeoutServed = yield* Deferred.make(); + const install = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: HttpClient.make((request) => { + requests.push(request.url); + const isChecksums = request.url.endsWith("/SHA256SUMS"); + if (requests.length === 1) { + Deferred.doneUnsafe(gatewayTimeoutServed, Effect.void); + return Effect.succeed( + HttpClientResponse.fromWeb(request, new Response("", { status: 504 })), + ); + } + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(isChecksums ? checksums : archiveBytes), + ), + ); + }), + runner: extractingRunner(fs, path), + validate: () => Effect.void, + }).pipe(Effect.forkScoped); + yield* Deferred.await(gatewayTimeoutServed); + yield* TestClock.adjust(Duration.seconds(2)); + const paths = yield* Fiber.join(install); + assert.equal(yield* fs.readFileString(paths.sentinelPath), `${version}\n`); + assert.equal(requests.filter((url) => url.endsWith("/SHA256SUMS")).length, 2); + }), + ); + + it.effect("retries a release archive whose body fails after the host answered", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-retry-body-" }); + const checksums = yield* validChecksums; + let archiveRequests = 0; + const archiveRequested = yield* Deferred.make(); + const install = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: HttpClient.make((request) => { + const isChecksums = request.url.endsWith("/SHA256SUMS"); + const response = HttpClientResponse.fromWeb( + request, + new Response(isChecksums ? checksums : archiveBytes), + ); + if (isChecksums || ++archiveRequests > 1) return Effect.succeed(response); + const failingBody = Stream.concat( + Stream.make(archiveBytes.slice(0, 4)), + Stream.fail( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.DecodeError({ request, response }), + }), + ), + ); + Deferred.doneUnsafe(archiveRequested, Effect.void); + // TestClock follows the work a woken sleep starts, so one adjust reaches the retry wait. + return Effect.sleep("1 second").pipe( + Effect.as( + Object.create(response, { stream: { value: failingBody } }), + ), + ); + }), + runner: extractingRunner(fs, path), + validate: () => Effect.void, + }).pipe(Effect.forkScoped); + yield* Deferred.await(archiveRequested); + yield* TestClock.adjust(Duration.seconds(3)); + const paths = yield* Fiber.join(install); + assert.equal(yield* fs.readFileString(paths.sentinelPath), `${version}\n`); + assert.equal(archiveRequests, 2); + }), + ); + + it.effect("fails at once when the release host cannot be reached", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-offline-" }); + let requests = 0; + const error = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version, + fs, + path, + platform: "linux", + arch: "x64", + httpClient: HttpClient.make((request) => { + requests += 1; + return Effect.fail( + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ request }), + }), + ); + }), + runner: extractingRunner(fs, path), + validate: () => Effect.die("must not validate without a download"), + }).pipe(Effect.flip); + assert.instanceOf(error, PinnedRuntimeInstallError); + assert.equal(requests, 1); }), ); diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index 77bcb39f09dcf..669ae01c40f59 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Option from "effect/Option"; +import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; import * as Semaphore from "effect/Semaphore"; import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"; @@ -119,6 +120,20 @@ interface PinnedRuntimeInstallInput { readonly onProgress?: (progress: PinnedRuntimeProgress) => void; } +// GitHub release downloads intermittently answer 5xx during incidents; a retry usually lands. +const DOWNLOAD_RETRY = { times: 3, schedule: Schedule.exponential("2 seconds") } as const; + +const isTransientDownloadFailure = ( + error: HttpClientError.HttpClientError | Cause.TimeoutError, +): boolean => { + // No answer or a stall means the host cannot reach the release source; retrying only delays that. + if (error._tag === "TimeoutError") return false; + // Body bytes are read raw, so a DecodeError is a body that failed after the host answered. + if (error.reason._tag === "DecodeError") return true; + const status = error.response?.status; + return status !== undefined && (status >= 500 || status === 429 || status === 408); +}; + const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")(function* ( httpClient: HttpClient.HttpClient, url: string, @@ -161,6 +176,7 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")( return bytes; }), ), + Effect.retry({ ...DOWNLOAD_RETRY, while: isTransientDownloadFailure }), Effect.mapError( (cause) => new PinnedRuntimeInstallError({ diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 184bd1deb485c..4cae117b2bcc6 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -184,21 +184,29 @@ it.layer(NodeServices.layer)("server self update", (it) => { ])("reports an HTTP $status downloading $asset and allows retry", ({ asset, status }) => Effect.gen(function* () { let failDownload = true; + const downloadFailed = yield* Deferred.make(); const { selfUpdate, order, baseDir } = yield* makeHarness({ httpClient: releaseHttpClient([], []).pipe( - HttpClient.transform((response, request) => - failDownload && request.url.endsWith("/SHA256SUMS") === (asset === "checksums") - ? Effect.succeed( - HttpClientResponse.fromWeb( - HttpClientRequest.get("https://user:password@cdn.example/private?token=secret"), - new Response("private response body", { status }), - ), - ) - : response, - ), + HttpClient.transform((response, request) => { + if (!failDownload || request.url.endsWith("/SHA256SUMS") !== (asset === "checksums")) + return response; + Deferred.doneUnsafe(downloadFailed, Effect.void); + return Effect.succeed( + HttpClientResponse.fromWeb( + HttpClientRequest.get("https://user:password@cdn.example/private?token=secret"), + new Response("private response body", { status }), + ), + ); + }), ), }); - const error = yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip); + const failing = yield* selfUpdate + .update({ targetVersion: "1.1.0" }) + .pipe(Effect.flip, Effect.forkScoped); + yield* Deferred.await(downloadFailed); + // A 5xx is retried after 2, 4, then 8 seconds before the update gives up. + yield* TestClock.adjust("14 seconds"); + const error = yield* Fiber.join(failing); expect(error.reason).toBe( `Could not prepare t3@1.1.0 while downloading the t3 release ${asset} (HTTP ${status} from cdn.example).`, );