Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 123 additions & 2 deletions apps/server/src/cloud/pinnedRuntime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Fiber from "effect/Fiber";
import * as Path from "effect/Path";
import * as Stream from "effect/Stream";
import * as NodePath from "node:path";
import * as TestClock from "effect/testing/TestClock";
import { HttpClient, HttpClientResponse } from "effect/http";
import { HttpClient, HttpClientError, HttpClientResponse } from "effect/http";
import * as ChildProcess from "effect/process/ChildProcess";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";

Expand Down Expand Up @@ -189,7 +190,127 @@ it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => {
validate: () => Effect.die("must not validate a missing archive"),
}).pipe(Effect.flip);
assert.isTrue(isUnpublishedRelease(yield* failsWith(404)));
assert.isFalse(isUnpublishedRelease(yield* failsWith(500)));
assert.isFalse(isUnpublishedRelease(yield* failsWith(403)));
}),
);

it.effect("retries a release download that GitHub answers with a gateway timeout", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-retry-" });
const checksums = yield* validChecksums;
const requests: string[] = [];
const gatewayTimeoutServed = yield* Deferred.make<void>();
const install = yield* ensurePinnedRuntimeInstalled({
baseDir,
version,
fs,
path,
platform: "linux",
arch: "x64",
httpClient: HttpClient.make((request) => {
requests.push(request.url);
const isChecksums = request.url.endsWith("/SHA256SUMS");
if (requests.length === 1) {
Deferred.doneUnsafe(gatewayTimeoutServed, Effect.void);
return Effect.succeed(
HttpClientResponse.fromWeb(request, new Response("", { status: 504 })),
);
}
return Effect.succeed(
HttpClientResponse.fromWeb(
request,
new Response(isChecksums ? checksums : archiveBytes),
),
);
}),
runner: extractingRunner(fs, path),
validate: () => Effect.void,
}).pipe(Effect.forkScoped);
yield* Deferred.await(gatewayTimeoutServed);
yield* TestClock.adjust(Duration.seconds(2));
const paths = yield* Fiber.join(install);
assert.equal(yield* fs.readFileString(paths.sentinelPath), `${version}\n`);
assert.equal(requests.filter((url) => url.endsWith("/SHA256SUMS")).length, 2);
}),
);

it.effect("retries a release archive whose body fails after the host answered", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-retry-body-" });
const checksums = yield* validChecksums;
let archiveRequests = 0;
const archiveRequested = yield* Deferred.make<void>();
const install = yield* ensurePinnedRuntimeInstalled({
baseDir,
version,
fs,
path,
platform: "linux",
arch: "x64",
httpClient: HttpClient.make((request) => {
const isChecksums = request.url.endsWith("/SHA256SUMS");
const response = HttpClientResponse.fromWeb(
request,
new Response(isChecksums ? checksums : archiveBytes),
);
if (isChecksums || ++archiveRequests > 1) return Effect.succeed(response);
const failingBody = Stream.concat(
Stream.make(archiveBytes.slice(0, 4)),
Stream.fail(
new HttpClientError.HttpClientError({
reason: new HttpClientError.DecodeError({ request, response }),
}),
),
);
Deferred.doneUnsafe(archiveRequested, Effect.void);
// TestClock follows the work a woken sleep starts, so one adjust reaches the retry wait.
return Effect.sleep("1 second").pipe(
Effect.as<HttpClientResponse.HttpClientResponse>(
Object.create(response, { stream: { value: failingBody } }),
),
);
}),
runner: extractingRunner(fs, path),
validate: () => Effect.void,
}).pipe(Effect.forkScoped);
yield* Deferred.await(archiveRequested);
yield* TestClock.adjust(Duration.seconds(3));
const paths = yield* Fiber.join(install);
assert.equal(yield* fs.readFileString(paths.sentinelPath), `${version}\n`);
assert.equal(archiveRequests, 2);
}),
);

it.effect("fails at once when the release host cannot be reached", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-offline-" });
let requests = 0;
const error = yield* ensurePinnedRuntimeInstalled({
baseDir,
version,
fs,
path,
platform: "linux",
arch: "x64",
httpClient: HttpClient.make((request) => {
requests += 1;
return Effect.fail(
new HttpClientError.HttpClientError({
reason: new HttpClientError.TransportError({ request }),
}),
);
}),
runner: extractingRunner(fs, path),
validate: () => Effect.die("must not validate without a download"),
}).pipe(Effect.flip);
assert.instanceOf(error, PinnedRuntimeInstallError);
assert.equal(requests, 1);
}),
);

Expand Down
16 changes: 16 additions & 0 deletions apps/server/src/cloud/pinnedRuntime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import * as Schema from "effect/Schema";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import * as Semaphore from "effect/Semaphore";
import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
Expand Down Expand Up @@ -119,6 +120,20 @@ interface PinnedRuntimeInstallInput {
readonly onProgress?: (progress: PinnedRuntimeProgress) => void;
}

// GitHub release downloads intermittently answer 5xx during incidents; a retry usually lands.
const DOWNLOAD_RETRY = { times: 3, schedule: Schedule.exponential("2 seconds") } as const;

const isTransientDownloadFailure = (
error: HttpClientError.HttpClientError | Cause.TimeoutError,
): boolean => {
// No answer or a stall means the host cannot reach the release source; retrying only delays that.
if (error._tag === "TimeoutError") return false;
// Body bytes are read raw, so a DecodeError is a body that failed after the host answered.
if (error.reason._tag === "DecodeError") return true;
const status = error.response?.status;
return status !== undefined && (status >= 500 || status === 429 || status === 408);
};

const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")(function* (
httpClient: HttpClient.HttpClient,
url: string,
Expand Down Expand Up @@ -161,6 +176,7 @@ const fetchReleaseAsset = Effect.fn("cloud.pinned_runtime.fetch_release_asset")(
return bytes;
}),
),
Effect.retry({ ...DOWNLOAD_RETRY, while: isTransientDownloadFailure }),
Effect.mapError(
(cause) =>
new PinnedRuntimeInstallError({
Expand Down
30 changes: 19 additions & 11 deletions apps/server/src/cloud/selfUpdate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,21 +184,29 @@ it.layer(NodeServices.layer)("server self update", (it) => {
])("reports an HTTP $status downloading $asset and allows retry", ({ asset, status }) =>
Effect.gen(function* () {
let failDownload = true;
const downloadFailed = yield* Deferred.make<void>();
const { selfUpdate, order, baseDir } = yield* makeHarness({
httpClient: releaseHttpClient([], []).pipe(
HttpClient.transform((response, request) =>
failDownload && request.url.endsWith("/SHA256SUMS") === (asset === "checksums")
? Effect.succeed(
HttpClientResponse.fromWeb(
HttpClientRequest.get("https://user:password@cdn.example/private?token=secret"),
new Response("private response body", { status }),
),
)
: response,
),
HttpClient.transform((response, request) => {
if (!failDownload || request.url.endsWith("/SHA256SUMS") !== (asset === "checksums"))
return response;
Deferred.doneUnsafe(downloadFailed, Effect.void);
return Effect.succeed(
HttpClientResponse.fromWeb(
HttpClientRequest.get("https://user:password@cdn.example/private?token=secret"),
new Response("private response body", { status }),
),
);
}),
),
});
const error = yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip);
const failing = yield* selfUpdate
.update({ targetVersion: "1.1.0" })
.pipe(Effect.flip, Effect.forkScoped);
yield* Deferred.await(downloadFailed);
// A 5xx is retried after 2, 4, then 8 seconds before the update gives up.
yield* TestClock.adjust("14 seconds");
const error = yield* Fiber.join(failing);
expect(error.reason).toBe(
`Could not prepare t3@1.1.0 while downloading the t3 release ${asset} (HTTP ${status} from cdn.example).`,
);
Expand Down
Loading