diff --git a/docs/architecture/desktop-downloads.md b/docs/architecture/desktop-downloads.md new file mode 100644 index 0000000..8722fa3 --- /dev/null +++ b/docs/architecture/desktop-downloads.md @@ -0,0 +1,60 @@ +# Desktop downloads + +`GET /api/download/` (`functions/api/download/[asset].ts`) counts a download +intent and redirects to the Stable installer for one platform. It never answers with +an error page: when no installer can be resolved it redirects to +`https://github.com/ScientFactory/scient-desktop/releases/latest`, which lists every +installer, and records `download_failed` with the reason, GitHub's HTTP status or error +name, elapsed time, attempts, and `fallback: "release_page"`. + +## Where the release comes from + +`resolveLatestDesktopRelease` (`functions/_lib/latest-release.ts`) tries, in order: + +1. The data-center edge cache (five minutes). +2. **ScientFactory's mirror**, while the release mirror worker has confirmed it within the + last six hours. Installers redirect to `downloads.scientfactory.com`. GitHub is not + contacted. +3. **GitHub's latest release handoff** + (`releases/latest/download/scient-release-handoff.json`). A missing response, timeout or + 5xx is retried once; a 4xx, including a 403/429 refusal, is not. Each attempt has four + seconds. +4. The mirror however old it is, when GitHub fails. + +Every source is validated by `parseReleaseHandoff` (`src/lib/release-handoff.ts`): the +official provenance repository, the four required installers, sizes and SHA-256 digests. +Redirect targets must pass `isOfficialDesktopReleaseDownload`: the canonical GitHub release +download path, or `downloads.scientfactory.com/scient-desktop/v/`. + +## The mirror + +The `scientfactory-release-mirror` worker (`workers/release-mirror/`) runs every ten minutes +against the `scient-downloads` R2 bucket, the same bucket that serves Scient Agent's copies. +Each run: + +1. Reads GitHub's latest handoff and validates it. +2. For each required installer, keeps an existing `scient-desktop//` whose size + and recorded SHA-256 match. A missing installer is streamed from the GitHub release into + R2 with the handoff's SHA-256 and `If-None-Match: *`; R2 refuses the write when the bytes + do not match. An existing installer with a different digest is never replaced: the run + fails, because the current pointer and year-long caches may still serve those bytes. A + changed installer needs a new release tag. +3. Writes `scient-desktop/stable.json` last, as a compare-and-swap on the pointer version the + run observed at its start: the handoff, GitHub's Last-Modified, `mirroredAt`, and + `checkedAt`. If another run wrote the pointer meanwhile, this run reports `superseded` + and leaves it, so a slow run that saw an older release cannot undo a newer promotion. + +A failed run leaves the previous pointer untouched, so the pointer only names installers +that are in the bucket. Older versions stay in the bucket. + +Deploy the worker with `bun run mirror:deploy`. The Pages project reads the bucket through +its `DESKTOP_DOWNLOADS` binding in `wrangler.jsonc`. + +## Why + +Before this, every cold edge cache asked GitHub for the handoff while the visitor waited, +and any refusal or error ended in a JSON 503. In November 2026 roughly a quarter of the +download attempts on some days failed that way, each in about 0.3 seconds, during bursts +of simultaneous requests against an expired cache. GitHub's status was not recorded, so +the specific refusal is unknown; the mirror and the release-page fallback remove the +dependency rather than tune around it. diff --git a/functions/_lib/events.ts b/functions/_lib/events.ts index 1c8b150..e99b289 100644 --- a/functions/_lib/events.ts +++ b/functions/_lib/events.ts @@ -24,6 +24,14 @@ export interface SiteEvent { readonly destinationPath?: string | null; readonly failureStage?: string | null; readonly failureReason?: string | null; + /** Download failures: GitHub's final HTTP status, or null when none arrived. */ + readonly upstreamStatus?: number | null; + /** Download failures: the error name when GitHub sent no usable response. */ + readonly upstreamError?: string | null; + readonly upstreamElapsedMs?: number | null; + readonly upstreamAttempts?: number | null; + /** Download failures: where the visitor was sent instead of an installer. */ + readonly fallback?: "release_page" | null; readonly sessionId?: string | null; } @@ -110,7 +118,12 @@ export async function insertSiteEvent( destination_path: cleanPath(event.destinationPath), failure_stage: limited(event.failureStage, 80), failure_reason: limited(event.failureReason, 120), - }).filter((entry): entry is [string, string] => entry[1] !== null), + upstream_status: event.upstreamStatus ?? null, + upstream_error: limited(event.upstreamError, 60), + upstream_elapsed_ms: event.upstreamElapsedMs ?? null, + upstream_attempts: event.upstreamAttempts ?? null, + fallback: event.fallback ?? null, + }).filter((entry): entry is [string, string | number] => entry[1] !== null), ); const statements: D1PreparedStatement[] = []; diff --git a/functions/_lib/latest-release.test.ts b/functions/_lib/latest-release.test.ts index d8f6658..78d6bad 100644 --- a/functions/_lib/latest-release.test.ts +++ b/functions/_lib/latest-release.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LatestReleaseResolutionError, resolveLatestDesktopRelease } from "./latest-release"; @@ -24,21 +24,73 @@ const handoffFixture = { ], }; +const mirroredHandoff = { + ...handoffFixture, + version: "0.6.4", + tag: "v0.6.4", + assets: handoffFixture.assets.map((asset) => ({ + ...asset, + name: asset.name.replace("0.6.5", "0.6.4"), + })), +}; + +const NOW = Date.parse("2026-11-13T12:00:00.000Z"); + function handoffResponse(headers?: HeadersInit): Response { return Response.json(handoffFixture, { headers: { "Last-Modified": "Sat, 22 Aug 2026 01:50:58 GMT", ...headers }, }); } -function context() { +function mirrorBucket(checkedAt: string | null, body?: string): R2Bucket { + const text = + body ?? + JSON.stringify({ + schemaVersion: 1, + handoff: mirroredHandoff, + publishedAt: "2026-11-01T00:00:00.000Z", + mirroredAt: "2026-11-01T00:05:00.000Z", + checkedAt, + }); + return { + get: vi.fn(async () => + checkedAt === null && body === undefined + ? null + : { size: text.length, text: async () => text }, + ), + } as unknown as R2Bucket; +} + +function context(bucket?: R2Bucket) { const pending: Array> = []; return { request: new Request("https://scientfactory.com/api/releases/latest"), + env: bucket ? { DESKTOP_DOWNLOADS: bucket } : {}, waitUntil: (promise: Promise) => pending.push(promise), pending, }; } +function emptyCache() { + const put = vi.fn(); + vi.stubGlobal("caches", { + default: { match: vi.fn().mockResolvedValue(undefined), put, delete: vi.fn() }, + }); + return put; +} + +function upstreamError(upstream: object) { + return expect.objectContaining>({ + stage: "release_fetch", + reason: "upstream_unavailable", + upstream: expect.objectContaining(upstream), + }); +} + +beforeEach(() => { + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); @@ -71,16 +123,98 @@ describe("latest release resolver", () => { expect(new URL(key.url).searchParams.toString()).toBe( new URLSearchParams({ source: "ScientFactory/scient-desktop", - schema: "handoff-v1", + schema: "handoff-v2", }).toString(), ); }); + it("serves a fresh mirror without contacting GitHub", async () => { + vi.useFakeTimers({ now: NOW }); + emptyCache(); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + const release = await resolveLatestDesktopRelease( + context(mirrorBucket("2026-11-13T11:55:00.000Z")), + ); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(release.tag_name).toBe("v0.6.4"); + expect(release.assets[0]?.browser_download_url).toBe( + "https://downloads.scientfactory.com/scient-desktop/v0.6.4/Scient-0.6.4-arm64.dmg", + ); + }); + + it("prefers GitHub over a mirror the worker has not confirmed recently", async () => { + vi.useFakeTimers({ now: NOW }); + emptyCache(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(handoffResponse())); + + const release = await resolveLatestDesktopRelease( + context(mirrorBucket("2026-11-13T05:00:00.000Z")), + ); + + expect(release.tag_name).toBe("v0.6.5"); + expect(release.assets[0]?.browser_download_url).toContain("github.com"); + }); + + it("falls back to a stale mirror when GitHub refuses, without retrying the refusal", async () => { + vi.useFakeTimers({ now: NOW }); + emptyCache(); + const fetchMock = vi + .fn() + .mockResolvedValue(new Response("Too many requests", { status: 429 })); + vi.stubGlobal("fetch", fetchMock); + + const release = await resolveLatestDesktopRelease( + context(mirrorBucket("2026-11-01T00:05:00.000Z")), + ); + + expect(release.tag_name).toBe("v0.6.4"); + expect(release.assets[0]?.browser_download_url).toContain("downloads.scientfactory.com"); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("ignores an unreadable mirror pointer and resolves from GitHub", async () => { + emptyCache(); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(handoffResponse())); + + await expect( + resolveLatestDesktopRelease(context(mirrorBucket(null, '{"schemaVersion":1}'))), + ).resolves.toMatchObject({ tag_name: "v0.6.5" }); + }); + + it("retries a GitHub server error once", async () => { + vi.useFakeTimers(); + emptyCache(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 502 })) + .mockResolvedValueOnce(handoffResponse()); + vi.stubGlobal("fetch", fetchMock); + + const pending = resolveLatestDesktopRelease(context()); + await vi.advanceTimersByTimeAsync(250); + + await expect(pending).resolves.toMatchObject({ tag_name: "v0.6.5" }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("reports GitHub's refusal with its status when there is no mirror", async () => { + const put = emptyCache(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(new Response(null, { status: 429 })), + ); + + await expect(resolveLatestDesktopRelease(context())).rejects.toEqual( + upstreamError({ status: 429, error: null, attempts: 1 }), + ); + expect(put).not.toHaveBeenCalled(); + }); + it("rejects an oversized handoff before parsing or caching it", async () => { - const put = vi.fn(); - vi.stubGlobal("caches", { - default: { match: vi.fn().mockResolvedValue(undefined), put, delete: vi.fn() }, - }); + const put = emptyCache(); vi.stubGlobal( "fetch", vi @@ -97,34 +231,24 @@ describe("latest release resolver", () => { expect(put).not.toHaveBeenCalled(); }); - it("aborts a stalled handoff request at the bounded timeout", async () => { + it("aborts each stalled attempt at the bounded timeout and names the timeout", async () => { vi.useFakeTimers(); - vi.stubGlobal("caches", { - default: { - match: vi.fn().mockResolvedValue(undefined), - put: vi.fn(), - delete: vi.fn(), - }, - }); - vi.stubGlobal( - "fetch", - vi.fn().mockImplementation( - (_input, init) => - new Promise((_resolve, reject) => { - init?.signal?.addEventListener("abort", () => reject(new DOMException("Aborted"))); - }), - ), + emptyCache(); + const fetchMock = vi.fn().mockImplementation( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new DOMException("Aborted"))); + }), ); + vi.stubGlobal("fetch", fetchMock); const pending = resolveLatestDesktopRelease(context()); const rejection = expect(pending).rejects.toEqual( - expect.objectContaining>({ - stage: "release_fetch", - reason: "upstream_unavailable", - }), + upstreamError({ status: null, error: "TimeoutError", attempts: 2 }), ); - await vi.advanceTimersByTimeAsync(8_000); + await vi.advanceTimersByTimeAsync(4_000 + 250 + 4_000); await rejection; + expect(fetchMock).toHaveBeenCalledTimes(2); }); it("discards an invalid cached value and replaces it from the handoff", async () => { diff --git a/functions/_lib/latest-release.ts b/functions/_lib/latest-release.ts index 9d93910..c6d9d0b 100644 --- a/functions/_lib/latest-release.ts +++ b/functions/_lib/latest-release.ts @@ -8,21 +8,39 @@ import { DESKTOP_RELEASE_REPOSITORY, GITHUB_RELEASE_HANDOFF_URL, } from "../../src/lib/release-source"; +import { readMirroredRelease, type MirroredRelease } from "./release-mirror"; const CACHE_CONTROL = "public, max-age=300"; -const CACHE_SCHEMA = "handoff-v1"; +const CACHE_SCHEMA = "handoff-v2"; const MAX_HANDOFF_BYTES = 64 * 1024; -const UPSTREAM_TIMEOUT_MS = 8_000; +const ATTEMPT_TIMEOUT_MS = 4_000; +const RETRY_DELAY_MS = 250; +const MAX_ATTEMPTS = 2; +// The mirror worker re-checks GitHub every ten minutes. A pointer it has not confirmed for this +// long may be behind a newer release, so GitHub is asked first and the pointer is the fallback. +const MIRROR_FRESH_MS = 6 * 60 * 60 * 1000; -interface ReleaseResolutionContext { +export interface ReleaseResolutionContext { readonly request: Request; + readonly env: { readonly DESKTOP_DOWNLOADS?: R2Bucket }; waitUntil(promise: Promise): void; } +/** What GitHub did on the last attempt, so a failure says more than "unavailable". */ +export interface UpstreamFailure { + /** HTTP status of the final response; null when no response arrived. */ + readonly status: number | null; + /** Error name when no usable response arrived, e.g. TimeoutError or TypeError. */ + readonly error: string | null; + readonly elapsedMs: number; + readonly attempts: number; +} + export class LatestReleaseResolutionError extends Error { constructor( readonly stage: "release_fetch" | "release_validation", readonly reason: "upstream_unavailable" | "release_metadata_invalid", + readonly upstream: UpstreamFailure | null = null, ) { super(reason); } @@ -45,7 +63,8 @@ function cachedResponse(release: Release): Response { }); } -async function readHandoff(response: Response): Promise { +/** Reads a handoff body within the size limit; shared with the release mirror worker. */ +export async function readHandoff(response: Response): Promise { const declaredLength = response.headers.get("Content-Length"); if (declaredLength !== null) { const length = Number(declaredLength); @@ -65,56 +84,122 @@ async function readHandoff(response: Response): Promise { } } -export async function resolveLatestDesktopRelease( - context: ReleaseResolutionContext, -): Promise { - const key = cacheKey(context.request); - const cached = await caches.default.match(key); - if (cached) { - try { - return parseRelease(await cached.json()); - } catch { - context.waitUntil(caches.default.delete(key)); - } - } +function errorName(error: unknown): string { + return error instanceof Error ? error.name.slice(0, 60) : "unknown"; +} + +type Attempt = + | { readonly release: Release } + | { readonly status: number | null; readonly error: string | null }; +async function attemptGitHub(): Promise { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), UPSTREAM_TIMEOUT_MS); - let upstream: Response; + const timeout = setTimeout(() => controller.abort(), ATTEMPT_TIMEOUT_MS); try { - upstream = await fetch(GITHUB_RELEASE_HANDOFF_URL, { + const upstream = await fetch(GITHUB_RELEASE_HANDOFF_URL, { headers: { Accept: "application/octet-stream, application/json", "User-Agent": "ScientFactory-download-service", }, signal: controller.signal, }); - } catch { + if (!upstream.ok) { + await upstream.body?.cancel(); + return { status: upstream.status, error: null }; + } + const handoff = await readHandoff(upstream); + try { + return { release: releaseFromHandoff(handoff, upstream.headers.get("Last-Modified")) }; + } catch { + throw new LatestReleaseResolutionError("release_validation", "release_metadata_invalid"); + } + } catch (error) { + if (error instanceof LatestReleaseResolutionError) throw error; + return { status: null, error: controller.signal.aborted ? "TimeoutError" : errorName(error) }; + } finally { clearTimeout(timeout); - throw new LatestReleaseResolutionError("release_fetch", "upstream_unavailable"); } +} - if (!upstream.ok) { - clearTimeout(timeout); - throw new LatestReleaseResolutionError("release_fetch", "upstream_unavailable"); +/** + * One retry for failures that may pass (no response, timeouts, 5xx). A 4xx, including GitHub's + * 403/429 refusals, is not retried: asking again within a second does not change the answer. + */ +async function fetchGitHubRelease(): Promise { + const started = Date.now(); + for (let attempts = 1; ; attempts += 1) { + const result = await attemptGitHub(); + if ("release" in result) return result.release; + const transient = result.status === null || result.status >= 500; + if (!transient || attempts >= MAX_ATTEMPTS) { + throw new LatestReleaseResolutionError("release_fetch", "upstream_unavailable", { + ...result, + elapsedMs: Date.now() - started, + attempts, + }); + } + await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS)); } +} +async function readMirror(context: ReleaseResolutionContext): Promise { + const bucket = context.env.DESKTOP_DOWNLOADS; + if (!bucket) return null; try { - const release = releaseFromHandoff( - await readHandoff(upstream), - upstream.headers.get("Last-Modified"), - ); - context.waitUntil(caches.default.put(key, cachedResponse(release))); - return release; + return await readMirroredRelease(bucket); } catch (error) { - if (error instanceof LatestReleaseResolutionError) throw error; - if (controller.signal.aborted) { - throw new LatestReleaseResolutionError("release_fetch", "upstream_unavailable"); + console.error( + JSON.stringify({ + message: "Desktop mirror pointer could not be read", + error: errorName(error), + detail: error instanceof Error ? error.message.slice(0, 200) : String(error), + }), + ); + return null; + } +} + +/** + * Resolution order: the edge cache, then ScientFactory's verified mirror while it is fresh, + * then GitHub's latest handoff, then the mirror however old it is. GitHub is only on the + * download path when the mirror is missing or stale. + */ +export async function resolveLatestDesktopRelease( + context: ReleaseResolutionContext, +): Promise { + const key = cacheKey(context.request); + const cached = await caches.default.match(key); + if (cached) { + try { + return parseRelease(await cached.json()); + } catch { + context.waitUntil(caches.default.delete(key)); } - throw new LatestReleaseResolutionError("release_validation", "release_metadata_invalid"); - } finally { - clearTimeout(timeout); } + + const mirror = await readMirror(context); + const mirrorIsFresh = mirror !== null && Date.now() - mirror.checkedAtMs <= MIRROR_FRESH_MS; + let release = mirrorIsFresh + ? null + : await fetchGitHubRelease().catch((error: unknown) => { + if (mirror === null) throw error; + console.error( + JSON.stringify({ + message: "GitHub release metadata failed; serving the last mirrored release", + mirroredTag: mirror.handoff.tag, + checkedAt: mirror.pointer.checkedAt, + ...(error instanceof LatestReleaseResolutionError + ? { reason: error.reason, upstream: error.upstream } + : { error: errorName(error) }), + }), + ); + return null; + }); + // Reaching here without a GitHub release means the mirror exists: GitHub failures without one rethrow. + release ??= releaseFromHandoff(mirror!.pointer.handoff, mirror!.pointer.publishedAt, "mirror"); + + context.waitUntil(caches.default.put(key, cachedResponse(release))); + return release; } export function latestReleaseResponse(release: Release): Response { diff --git a/functions/_lib/release-mirror.ts b/functions/_lib/release-mirror.ts new file mode 100644 index 0000000..33825d1 --- /dev/null +++ b/functions/_lib/release-mirror.ts @@ -0,0 +1,86 @@ +// FILE: release-mirror.ts +// Purpose: Reads and writes the Stable pointer of ScientFactory's own copy of the desktop installers. +// Layer: Cloudflare utility shared by the Pages download path and the release mirror worker + +import { parseReleaseHandoff, type ReleaseHandoff } from "../../src/lib/release-handoff"; +import { DESKTOP_MIRROR_STABLE_KEY } from "../../src/lib/release-source"; + +const MAX_POINTER_BYTES = 128 * 1024; + +/** + * The Stable pointer. The mirror worker writes it only after every installer of `handoff` + * is in the bucket with R2-verified SHA-256, so a pointer always names complete bytes. + */ +export interface DesktopMirrorPointer { + readonly schemaVersion: 1; + /** The handoff exactly as GitHub served it, revalidated on every read. */ + readonly handoff: unknown; + /** GitHub's Last-Modified for the handoff, the closest thing to a publish time. */ + readonly publishedAt: string | null; + /** When the installers of this handoff finished copying. */ + readonly mirroredAt: string; + /** When the worker last confirmed this handoff is still GitHub's latest. */ + readonly checkedAt: string; +} + +export interface MirroredRelease { + readonly pointer: DesktopMirrorPointer; + readonly handoff: ReleaseHandoff; + readonly checkedAtMs: number; +} + +function timestamp(value: unknown): number | null { + if (typeof value !== "string") return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +export function parseMirrorPointer(value: unknown): MirroredRelease { + if (typeof value !== "object" || value === null) { + throw new Error("Mirror pointer is not an object."); + } + const pointer = value as Partial; + const checkedAtMs = timestamp(pointer.checkedAt); + if ( + pointer.schemaVersion !== 1 || + (pointer.publishedAt !== null && typeof pointer.publishedAt !== "string") || + timestamp(pointer.mirroredAt) === null || + checkedAtMs === null + ) { + throw new Error("Mirror pointer is invalid."); + } + return { + pointer: pointer as DesktopMirrorPointer, + handoff: parseReleaseHandoff(pointer.handoff), + checkedAtMs, + }; +} + +/** The current Stable pointer, null when nothing has been mirrored yet. Throws when unreadable. */ +export async function readMirroredRelease(bucket: R2Bucket): Promise { + const object = await bucket.get(DESKTOP_MIRROR_STABLE_KEY); + if (object === null) return null; + if (object.size > MAX_POINTER_BYTES) throw new Error("Mirror pointer is too large."); + return parseMirrorPointer(JSON.parse(await object.text()) as unknown); +} + +/** + * Writes the pointer only if it is still the version this run observed (`observedEtag`, or + * absent when null). Returns false when another run wrote it first, so a slow run that saw an + * older release can never replace a newer promotion. + */ +export async function writeMirrorPointer( + bucket: R2Bucket, + pointer: DesktopMirrorPointer, + observedEtag: string | null, +): Promise { + const written = await bucket.put(DESKTOP_MIRROR_STABLE_KEY, JSON.stringify(pointer), { + onlyIf: + observedEtag === null ? new Headers({ "If-None-Match": "*" }) : { etagMatches: observedEtag }, + httpMetadata: { + contentType: "application/json; charset=utf-8", + cacheControl: "no-store", + }, + }); + return written !== null; +} diff --git a/functions/api/download/[asset].test.ts b/functions/api/download/[asset].test.ts index e67fafc..2d2abc6 100644 --- a/functions/api/download/[asset].test.ts +++ b/functions/api/download/[asset].test.ts @@ -52,7 +52,7 @@ function handoffResponse(value: unknown = handoffFixture): Response { } function createDatabase(run = vi.fn().mockResolvedValue({ success: true })) { - const bind = vi.fn(() => ({ run })); + const bind = vi.fn((..._values: unknown[]) => ({ run })); const prepare = vi.fn(() => ({ bind })); const batch = vi.fn(async (statements: ReadonlyArray<{ run?: () => Promise }>) => { await Promise.all(statements.map((statement) => statement.run?.())); @@ -142,36 +142,65 @@ describe("tracked download redirect", () => { await flushWaitUntil(context); }); - it("records a ScientFactory-side failure when resolution fails", async () => { + it("sends a visitor to the release page and records why when resolution fails", async () => { vi.spyOn(console, "error").mockImplementation(() => undefined); const db = createDatabase(); const context = createContext("macArm64", db.database); vi.stubGlobal( "fetch", - vi.fn().mockResolvedValue(new Response(null, { status: 503 })), + vi.fn().mockResolvedValue(new Response(null, { status: 429 })), ); const response = await onRequestGet(context); - expect(response.status).toBe(503); + expect(response.status).toBe(302); + expect(response.headers.get("Location")).toBe( + "https://github.com/ScientFactory/scient-desktop/releases/latest", + ); await flushWaitUntil(context); - expect(db.bind).toHaveBeenCalledWith( - expect.any(String), - "download_failed", - "diagnostic", - expect.any(String), - expect.stringMatching(/^web-event:/), - JSON.stringify({ - page_path: "/download", - asset_key: "macArm64", - failure_stage: "release_fetch", - failure_reason: "upstream_unavailable", - }), - "event", - expect.stringMatching(/^web-event:/), - null, - "essential", + const failure = db.bind.mock.calls.find((call) => call[1] === "download_failed"); + expect(failure?.[2]).toBe("diagnostic"); + expect(JSON.parse(String(failure?.[5]))).toEqual({ + page_path: "/download", + asset_key: "macArm64", + failure_stage: "release_fetch", + failure_reason: "upstream_unavailable", + upstream_status: 429, + upstream_elapsed_ms: expect.any(Number), + upstream_attempts: 1, + fallback: "release_page", + }); + }); + + it("redirects to ScientFactory's mirrored installer when the mirror is current", async () => { + const db = createDatabase(); + const context = createContext("linuxX64", db.database); + const pointer = JSON.stringify({ + schemaVersion: 1, + handoff: handoffFixture, + publishedAt: null, + mirroredAt: new Date().toISOString(), + checkedAt: new Date().toISOString(), + }); + Object.assign(context.env, { + DESKTOP_DOWNLOADS: { get: async () => ({ size: pointer.length, text: async () => pointer }) }, + }); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + + const response = await onRequestGet(context); + + expect(response.status).toBe(302); + expect(response.headers.get("Location")).toBe( + "https://downloads.scientfactory.com/scient-desktop/v0.5.7/Scient-0.5.7-x86_64.AppImage", ); + expect(fetchMock).not.toHaveBeenCalled(); + await flushWaitUntil(context); + const click = db.bind.mock.calls.find((call) => call[1] === "download_clicked"); + expect(JSON.parse(String(click?.[5]))).toMatchObject({ + destination_host: "downloads.scientfactory.com", + destination_path: "/scient-desktop/v0.5.7/Scient-0.5.7-x86_64.AppImage", + }); }); it("rejects unknown asset keys before contacting GitHub", async () => { diff --git a/functions/api/download/[asset].ts b/functions/api/download/[asset].ts index fb52ad0..9c7ca91 100644 --- a/functions/api/download/[asset].ts +++ b/functions/api/download/[asset].ts @@ -1,14 +1,18 @@ // FILE: [asset].ts -// Purpose: Counts a download intent, then redirects to the matching official GitHub installer. +// Purpose: Counts a download intent, then redirects to the matching official installer. // Layer: Cloudflare Pages Function import { findDownloadAsset, type DownloadAssetKey } from "../../../src/lib/download-assets"; import type { Release, ReleaseAsset } from "../../../src/lib/release-schema"; -import { isOfficialDesktopReleaseDownload } from "../../../src/lib/release-source"; +import { + DESKTOP_LATEST_RELEASE_PAGE_URL, + isOfficialDesktopReleaseDownload, +} from "../../../src/lib/release-source"; import { queueSiteEvent } from "../../_lib/events"; import { LatestReleaseResolutionError, resolveLatestDesktopRelease, + type UpstreamFailure, } from "../../_lib/latest-release"; const DOWNLOAD_ASSET_KEYS = new Set([ @@ -22,6 +26,7 @@ class DownloadResolutionError extends Error { constructor( readonly stage: string, readonly reason: string, + readonly upstream: UpstreamFailure | null = null, ) { super(reason); } @@ -44,7 +49,7 @@ async function resolveDownload( release = await resolveLatestDesktopRelease(context); } catch (error) { if (error instanceof LatestReleaseResolutionError) { - throw new DownloadResolutionError(error.stage, error.reason); + throw new DownloadResolutionError(error.stage, error.reason, error.upstream); } throw new DownloadResolutionError("release_fetch", "upstream_request_failed"); } @@ -60,12 +65,12 @@ async function resolveDownload( return { release, asset }; } -function redirectResponse(asset: ReleaseAsset): Response { +function redirectResponse(location: string): Response { return new Response(null, { status: 302, headers: { "Cache-Control": "no-store", - Location: asset.browser_download_url, + Location: location, "Referrer-Policy": "no-referrer", "X-Content-Type-Options": "nosniff", "X-Robots-Tag": "noindex, nofollow", @@ -73,25 +78,6 @@ function redirectResponse(asset: ReleaseAsset): Response { }); } -function unavailableResponse(): Response { - return Response.json( - { - error: - "This installer is temporarily unavailable. Please return to the download page and try again.", - }, - { - status: 503, - headers: { - "Cache-Control": "no-store", - "Content-Type": "application/json; charset=utf-8", - "Retry-After": "60", - "X-Content-Type-Options": "nosniff", - "X-Robots-Tag": "noindex, nofollow", - }, - }, - ); -} - async function handleDownload( context: EventContext, track: boolean, @@ -113,7 +99,7 @@ async function handleDownload( destinationPath: destination.pathname, }); } - return redirectResponse(asset); + return redirectResponse(asset.browser_download_url); } catch (error) { const failure = error instanceof DownloadResolutionError @@ -127,18 +113,25 @@ async function handleDownload( assetKey: key, failureStage: failure.stage, failureReason: failure.reason, + upstreamStatus: failure.upstream?.status ?? null, + upstreamError: failure.upstream?.error ?? null, + upstreamElapsedMs: failure.upstream?.elapsedMs ?? null, + upstreamAttempts: failure.upstream?.attempts ?? null, + fallback: "release_page", }); } console.error( JSON.stringify({ - message: "Download redirect could not be resolved", + message: "Download redirect could not be resolved; sent to the release page", assetKey: key, stage: failure.stage, reason: failure.reason, + upstream: failure.upstream, }), ); - return unavailableResponse(); + // The release page lists every installer, so a visitor still reaches a download. + return redirectResponse(DESKTOP_LATEST_RELEASE_PAGE_URL); } } diff --git a/functions/api/releases/latest.test.ts b/functions/api/releases/latest.test.ts index cea0953..e1946ae 100644 --- a/functions/api/releases/latest.test.ts +++ b/functions/api/releases/latest.test.ts @@ -53,6 +53,7 @@ function handoffResponse(value: unknown = handoffFixture): Response { function createContext() { return { request: new Request("https://scientfactory.com/api/releases/latest?ignored=true"), + env: {}, waitUntil: vi.fn(), } as unknown as Parameters[0]; } diff --git a/package.json b/package.json index db5adc7..bf8b0b4 100644 --- a/package.json +++ b/package.json @@ -25,9 +25,13 @@ "events:types:check": "wrangler types workers/events/worker-configuration.d.ts --config workers/events/wrangler.jsonc --env-interface AnalyticsWorkerBindings --include-runtime false --check", "events:typecheck": "tsc --noEmit --project workers/events/tsconfig.json", "events:deploy": "wrangler deploy --config workers/events/wrangler.jsonc", + "mirror:types": "wrangler types workers/release-mirror/worker-configuration.d.ts --config workers/release-mirror/wrangler.jsonc --env-interface Env --include-runtime false", + "mirror:types:check": "wrangler types workers/release-mirror/worker-configuration.d.ts --config workers/release-mirror/wrangler.jsonc --env-interface Env --include-runtime false --check", + "mirror:typecheck": "tsc --noEmit --project workers/release-mirror/tsconfig.json", + "mirror:deploy": "wrangler deploy --config workers/release-mirror/wrangler.jsonc", "format": "oxfmt", "format:check": "oxfmt --check", - "check": "bun run format:check && bun run typecheck && bun run events:types:check && bun run events:typecheck && bun run test && bun run build && bun run build:edge" + "check": "bun run format:check && bun run typecheck && bun run events:types:check && bun run events:typecheck && bun run mirror:types:check && bun run mirror:typecheck && bun run test && bun run build && bun run build:edge" }, "dependencies": { "@fontsource-variable/dm-sans": "5.3.0", diff --git a/src/lib/release-handoff.test.ts b/src/lib/release-handoff.test.ts index bcdb3d3..f7264c7 100644 --- a/src/lib/release-handoff.test.ts +++ b/src/lib/release-handoff.test.ts @@ -92,6 +92,34 @@ describe("Scient release handoff", () => { expect(() => releaseFromHandoff(value, lastModified)).toThrow(); }); + it("points installers at the mirror and keeps every other asset on GitHub", () => { + const release = releaseFromHandoff( + { + ...handoffFixture, + assets: [ + ...handoffFixture.assets, + { name: "latest-mac.yml", size: 500, sha256: "9".repeat(64) }, + ], + }, + "2026-08-22T01:50:58.000Z", + "mirror", + ); + const urls = Object.fromEntries( + release.assets.map((asset) => [asset.name, asset.browser_download_url]), + ); + expect(urls["Scient-0.6.5-arm64.dmg"]).toBe( + "https://downloads.scientfactory.com/scient-desktop/v0.6.5/Scient-0.6.5-arm64.dmg", + ); + expect(urls["Scient-0.6.5-x86_64.AppImage"]).toBe( + "https://downloads.scientfactory.com/scient-desktop/v0.6.5/Scient-0.6.5-x86_64.AppImage", + ); + expect(urls["latest-mac.yml"]).toBe( + "https://github.com/ScientFactory/scient-desktop/releases/download/v0.6.5/latest-mac.yml", + ); + expect(urls["SHA256SUMS.txt"]).toContain("github.com"); + expect(release.published_at).toBe("2026-08-22T01:50:58.000Z"); + }); + it("keeps publication copy optional when the CDN omits its date", () => { expect(releaseFromHandoff(handoffFixture, null).published_at).toBeNull(); expect(releaseFromHandoff(handoffFixture, "not-a-date").published_at).toBeNull(); diff --git a/src/lib/release-handoff.ts b/src/lib/release-handoff.ts index 868b9f1..2a0a0c1 100644 --- a/src/lib/release-handoff.ts +++ b/src/lib/release-handoff.ts @@ -5,15 +5,26 @@ import type { Release, ReleaseAsset } from "./release-schema"; import { DESKTOP_RELEASE_REPOSITORY, + desktopMirrorUrl, isOfficialDesktopProvenanceRepository, } from "./release-source"; -interface HandoffAsset { +export interface ReleaseHandoffAsset { readonly name: string; readonly size: number; readonly sha256: string; } +/** The validated facts the website relies on from a release handoff. */ +export interface ReleaseHandoff { + readonly version: string; + readonly tag: string; + readonly assets: readonly ReleaseHandoffAsset[]; +} + +/** Where installer downloads point: GitHub, or ScientFactory's verified copy. */ +export type ReleaseDownloadHost = "github" | "mirror"; + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -25,7 +36,7 @@ function parsePublishedAt(value: string | null): string | null { return new Date(timestamp).toISOString(); } -function parseAsset(value: unknown): HandoffAsset { +function parseAsset(value: unknown): ReleaseHandoffAsset { if ( !isRecord(value) || typeof value.name !== "string" || @@ -42,7 +53,7 @@ function parseAsset(value: unknown): HandoffAsset { return { name: value.name, size: value.size, sha256: value.sha256 }; } -function contentType(name: string): string { +export function releaseAssetContentType(name: string): string { if (name.endsWith(".dmg")) return "application/x-apple-diskimage"; if (name.endsWith(".yml")) return "text/yaml"; if (name.endsWith(".txt")) return "text/plain"; @@ -51,17 +62,21 @@ function contentType(name: string): string { return "application/octet-stream"; } -function releaseAsset(repository: string, tag: string, asset: HandoffAsset): ReleaseAsset { - return { - name: asset.name, - browser_download_url: `https://github.com/${repository}/releases/download/${tag}/${encodeURIComponent(asset.name)}`, - content_type: contentType(asset.name), - size: asset.size, - digest: `sha256:${asset.sha256}`, - }; +/** The installers every Stable release must carry, one per download card. */ +export function requiredInstallerNames(version: string): readonly string[] { + return [ + `Scient-${version}-arm64.dmg`, + `Scient-${version}-x64.dmg`, + `Scient-${version}-x64.exe`, + `Scient-${version}-x86_64.AppImage`, + ]; } -export function releaseFromHandoff(value: unknown, lastModified: string | null): Release { +export function githubReleaseDownloadUrl(tag: string, name: string): string { + return `https://github.com/${DESKTOP_RELEASE_REPOSITORY}/releases/download/${tag}/${encodeURIComponent(name)}`; +} + +export function parseReleaseHandoff(value: unknown): ReleaseHandoff { if ( !isRecord(value) || value.schemaVersion !== 1 || @@ -88,24 +103,29 @@ export function releaseFromHandoff(value: unknown, lastModified: string | null): throw new Error("Release handoff contains duplicate assets."); } - const tag = `v${value.version}`; - const requiredInstallers = [ - `Scient-${value.version}-arm64.dmg`, - `Scient-${value.version}-x64.dmg`, - `Scient-${value.version}-x64.exe`, - `Scient-${value.version}-x86_64.AppImage`, - ]; const names = new Set(assets.map((asset) => asset.name)); - if (!requiredInstallers.every((name) => names.has(name))) { + if (!requiredInstallerNames(value.version).every((name) => names.has(name))) { throw new Error("Release handoff is missing a required installer."); } + + return { version: value.version, tag: `v${value.version}`, assets }; +} + +export function releaseFromHandoff( + value: unknown, + lastModified: string | null, + downloads: ReleaseDownloadHost = "github", +): Release { + const { version, tag, assets } = parseReleaseHandoff(value); + // Only the installers are mirrored; every other asset stays on GitHub. + const mirrored = new Set(downloads === "mirror" ? requiredInstallerNames(version) : []); const checksumSize = assets.reduce( (size, asset) => size + asset.sha256.length + 2 + asset.name.length + 1, 0, ); const checksumAsset: ReleaseAsset = { name: "SHA256SUMS.txt", - browser_download_url: `https://github.com/${DESKTOP_RELEASE_REPOSITORY}/releases/download/${tag}/SHA256SUMS.txt`, + browser_download_url: githubReleaseDownloadUrl(tag, "SHA256SUMS.txt"), content_type: "text/plain", size: checksumSize, }; @@ -117,7 +137,15 @@ export function releaseFromHandoff(value: unknown, lastModified: string | null): published_at: parsePublishedAt(lastModified), prerelease: false, assets: [ - ...assets.map((asset) => releaseAsset(DESKTOP_RELEASE_REPOSITORY, tag, asset)), + ...assets.map((asset): ReleaseAsset => ({ + name: asset.name, + browser_download_url: mirrored.has(asset.name) + ? desktopMirrorUrl(tag, asset.name) + : githubReleaseDownloadUrl(tag, asset.name), + content_type: releaseAssetContentType(asset.name), + size: asset.size, + digest: `sha256:${asset.sha256}`, + })), checksumAsset, ], }; diff --git a/src/lib/release-source.test.ts b/src/lib/release-source.test.ts index a2d504b..950f5ed 100644 --- a/src/lib/release-source.test.ts +++ b/src/lib/release-source.test.ts @@ -32,6 +32,25 @@ describe("Scient Desktop release source", () => { expect(isOfficialDesktopReleaseDownload(url)).toBe(false); }); + it("accepts ScientFactory's mirrored installer path", () => { + expect( + isOfficialDesktopReleaseDownload( + "https://downloads.scientfactory.com/scient-desktop/v0.6.28/Scient-0.6.28-arm64.dmg", + ), + ).toBe(true); + }); + + it.each([ + "http://downloads.scientfactory.com/scient-desktop/v0.6.28/Scient-0.6.28-arm64.dmg", + "https://downloads.scientfactory.com/scient-agent/v0.1.5/scient-agent-darwin-arm64", + "https://downloads.scientfactory.com/scient-desktop/stable.json", + "https://downloads.scientfactory.com/scient-desktop/v0.6.28/../stable.json", + "https://downloads.scientfactory.com/scient-desktop/v0.6.28/Scient.dmg?x=1", + "https://downloads.scientfactory.com.evil.test/scient-desktop/v0.6.28/Scient.dmg", + ])("rejects an untrusted mirror path: %s", (url) => { + expect(isOfficialDesktopReleaseDownload(url)).toBe(false); + }); + it("accepts the exact canonical release page", () => { expect( isOfficialDesktopReleasePage( diff --git a/src/lib/release-source.ts b/src/lib/release-source.ts index 03c9ad0..333eaac 100644 --- a/src/lib/release-source.ts +++ b/src/lib/release-source.ts @@ -13,33 +13,67 @@ const TRUSTED_PROVENANCE_REPOSITORIES = new Set([ export const GITHUB_RELEASE_API_URL = `https://api.github.com/repos/${DESKTOP_RELEASE_REPOSITORY}/releases/latest`; export const GITHUB_RELEASE_HANDOFF_URL = `https://github.com/${DESKTOP_RELEASE_REPOSITORY}/releases/latest/download/scient-release-handoff.json`; export const DESKTOP_REPOSITORY_URL = `https://github.com/${DESKTOP_RELEASE_REPOSITORY}`; +export const DESKTOP_LATEST_RELEASE_PAGE_URL = `${DESKTOP_REPOSITORY_URL}/releases/latest`; export const RELEASE_CACHE_NAMESPACE = `scient-latest-release-v3:${DESKTOP_RELEASE_REPOSITORY}`; +// ScientFactory's own copy of the Stable installers lives in the `scient-downloads` R2 bucket, +// served at downloads.scientfactory.com. The release mirror worker copies each installer to +// `scient-desktop//` and only then writes the Stable pointer. +export const DESKTOP_MIRROR_ORIGIN = "https://downloads.scientfactory.com"; +export const DESKTOP_MIRROR_PREFIX = "scient-desktop/"; +export const DESKTOP_MIRROR_STABLE_KEY = `${DESKTOP_MIRROR_PREFIX}stable.json`; + +export function desktopMirrorKey(tag: string, name: string): string { + return `${DESKTOP_MIRROR_PREFIX}${tag}/${name}`; +} + +export function desktopMirrorUrl(tag: string, name: string): string { + return `${DESKTOP_MIRROR_ORIGIN}/${DESKTOP_MIRROR_PREFIX}${tag}/${encodeURIComponent(name)}`; +} + export function isOfficialDesktopProvenanceRepository(value: string): boolean { return TRUSTED_PROVENANCE_REPOSITORIES.has(value); } -function trustedGitHubPath(value: string, segment: "download" | "tag"): boolean { +function plainHttpsUrl(value: string, hostname: string): URL | null { try { const destination = new URL(value); - if ( - destination.protocol !== "https:" || - destination.hostname !== "github.com" || - destination.port !== "" || - destination.username !== "" || - destination.password !== "" - ) { - return false; - } - const prefix = `/${DESKTOP_RELEASE_REPOSITORY}/releases/${segment}/`; - return destination.pathname.startsWith(prefix) && destination.pathname.length > prefix.length; + return destination.protocol === "https:" && + destination.hostname === hostname && + destination.port === "" && + destination.username === "" && + destination.password === "" + ? destination + : null; } catch { - return false; + return null; } } +function trustedGitHubPath(value: string, segment: "download" | "tag"): boolean { + const destination = plainHttpsUrl(value, "github.com"); + if (!destination) return false; + const prefix = `/${DESKTOP_RELEASE_REPOSITORY}/releases/${segment}/`; + return destination.pathname.startsWith(prefix) && destination.pathname.length > prefix.length; +} + +const MIRROR_INSTALLER_PATH = new RegExp( + `^/${DESKTOP_MIRROR_PREFIX}v\\d+\\.\\d+\\.\\d+/[A-Za-z0-9][A-Za-z0-9._+-]*$`, + "u", +); + +function trustedMirrorPath(value: string): boolean { + const destination = plainHttpsUrl(value, new URL(DESKTOP_MIRROR_ORIGIN).hostname); + return ( + destination !== null && + destination.search === "" && + destination.hash === "" && + MIRROR_INSTALLER_PATH.test(destination.pathname) + ); +} + export function isOfficialDesktopReleaseDownload(value: string): boolean { - return trustedGitHubPath(value, "download"); + return trustedGitHubPath(value, "download") || trustedMirrorPath(value); } export function isOfficialDesktopReleasePage(value: string): boolean { diff --git a/src/lib/releases.ts b/src/lib/releases.ts index aae48e8..dbf64b6 100644 --- a/src/lib/releases.ts +++ b/src/lib/releases.ts @@ -4,6 +4,7 @@ import { parseRelease, type Release, type ReleaseAsset } from "./release-schema"; import { + DESKTOP_LATEST_RELEASE_PAGE_URL, DESKTOP_REPOSITORY_URL, GITHUB_RELEASE_API_URL, RELEASE_CACHE_NAMESPACE, @@ -20,7 +21,7 @@ let inFlightRelease: Promise | null = null; export const REPO_URL = DESKTOP_REPOSITORY_URL; export const RELEASES_URL = `${REPO_URL}/releases`; -export const LATEST_RELEASE_URL = `${RELEASES_URL}/latest`; +export const LATEST_RELEASE_URL = DESKTOP_LATEST_RELEASE_PAGE_URL; interface CachedRelease { readonly cachedAt: number; diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 3d18b5d..449de16 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,8 +1,9 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --include-runtime=false` (hash: 814495ecf9de3577f42d97af8717dc0b) +// Generated by Wrangler by running `wrangler types --include-runtime=false` (hash: 4e346603558ee67bc29d862f854978a2) interface __BaseEnv_Env { + DESKTOP_DOWNLOADS: R2Bucket; DOWNLOAD_DB: D1Database; - IDENTITY_ANALYTICS_ENABLED: "false"; + IDENTITY_ANALYTICS_ENABLED: "true"; } declare namespace Cloudflare { interface Env extends __BaseEnv_Env {} diff --git a/workers/release-mirror/src/index.ts b/workers/release-mirror/src/index.ts new file mode 100644 index 0000000..bfecaac --- /dev/null +++ b/workers/release-mirror/src/index.ts @@ -0,0 +1,15 @@ +// FILE: index.ts +// Purpose: Keeps downloads.scientfactory.com's copy of the Stable desktop installers current. +// Layer: Cloudflare Worker (cron only; no public routes) + +import { syncDesktopMirror } from "./mirror"; + +export default { + async scheduled(controller, env) { + const result = await syncDesktopMirror( + env.DESKTOP_DOWNLOADS, + new Date(controller.scheduledTime), + ); + console.log(JSON.stringify({ message: "Desktop mirror synchronized", ...result })); + }, +} satisfies ExportedHandler; diff --git a/workers/release-mirror/src/mirror.test.ts b/workers/release-mirror/src/mirror.test.ts new file mode 100644 index 0000000..3549e7c --- /dev/null +++ b/workers/release-mirror/src/mirror.test.ts @@ -0,0 +1,333 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { parseMirrorPointer } from "../../../functions/_lib/release-mirror"; +import { syncDesktopMirror } from "./mirror"; + +interface HandoffFixture { + readonly schemaVersion: 1; + readonly product: "Scient"; + readonly version: string; + readonly tag: string; + readonly source: { readonly repository: string; readonly commit: string; readonly tree: string }; + readonly assets: ReadonlyArray<{ + readonly name: string; + readonly size: number; + readonly sha256: string; + }>; +} + +function handoffFor(version: string, digests = "1234"): HandoffFixture { + return { + schemaVersion: 1, + product: "Scient", + version, + tag: `v${version}`, + source: { + repository: "ScientFactory/scient-desktop", + commit: "a".repeat(40), + tree: "b".repeat(40), + }, + assets: [ + { name: `Scient-${version}-arm64.dmg`, size: 3, sha256: digests[0]!.repeat(64) }, + { name: `Scient-${version}-x64.dmg`, size: 3, sha256: digests[1]!.repeat(64) }, + { name: `Scient-${version}-x64.exe`, size: 3, sha256: digests[2]!.repeat(64) }, + { name: `Scient-${version}-x86_64.AppImage`, size: 3, sha256: digests[3]!.repeat(64) }, + { name: "latest.yml", size: 3, sha256: "5".repeat(64) }, + ], + }; +} + +const handoff = handoffFor("0.6.28"); +const NOW = new Date("2026-11-13T12:00:00.000Z"); +const HANDOFF_URL = + "https://github.com/ScientFactory/scient-desktop/releases/latest/download/scient-release-handoff.json"; +const STABLE = "scient-desktop/stable.json"; + +interface StoredObject { + readonly body: string; + readonly etag: string; + readonly customMetadata?: Record; +} + +/** + * An in-memory bucket with R2's write rules used here: a write whose bytes miss the declared + * SHA-256 is refused, and `If-None-Match: *` / `etagMatches` writes return null when they fail. + */ +function memoryBucket(initial: Record = {}) { + let version = 0; + const objects = new Map( + Object.entries(initial).map(([key, body]) => [key, { body, etag: `e${(version += 1)}` }]), + ); + const writes: string[] = []; + const describe = (object: StoredObject | undefined) => + object + ? { + size: object.body.length, + etag: object.etag, + customMetadata: object.customMetadata ?? {}, + text: async () => object.body, + } + : null; + const bucket = { + head: vi.fn(async (key: string) => describe(objects.get(key))), + get: vi.fn(async (key: string) => describe(objects.get(key))), + put: vi.fn(async (key: string, value: ReadableStream | string, options?: R2PutOptions) => { + const onlyIf = options?.onlyIf; + const existing = objects.get(key); + if (onlyIf instanceof Headers && onlyIf.get("If-None-Match") === "*" && existing) { + return null; + } + if (onlyIf && !(onlyIf instanceof Headers) && onlyIf.etagMatches !== existing?.etag) { + return null; + } + const body = typeof value === "string" ? value : await new Response(value).text(); + if (options?.sha256 !== undefined && options.sha256 !== `${body[0]}`.repeat(64)) { + throw new Error("put: The SHA-256 checksum you specified did not match what we received."); + } + const stored = { + body, + etag: `e${(version += 1)}`, + customMetadata: options?.customMetadata as Record | undefined, + }; + objects.set(key, stored); + writes.push(key); + return describe(stored); + }), + }; + return { bucket: bucket as unknown as R2Bucket, objects, writes, put: bucket.put }; +} + +/** + * GitHub's latest handoff is `latest()`; every tag's installers stay downloadable, each served as + * three copies of its digest's first character. + */ +function githubFetch( + latest: () => HandoffFixture = () => handoff, + overrides: Record Response | Promise> = {}, +) { + return vi.fn(async (input) => { + const url = String(input); + if (overrides[url]) return overrides[url]!(); + if (url === HANDOFF_URL) { + return Response.json(latest(), { + headers: { "Last-Modified": "Tue, 10 Nov 2026 16:14:08 GMT" }, + }); + } + const [tag = "", encodedName = ""] = url.split("/").slice(-2); + const release = latest().tag === tag ? latest() : handoffFor(tag.slice(1)); + const asset = release.assets.find( + (candidate) => candidate.name === decodeURIComponent(encodedName), + ); + if (!asset) return new Response(null, { status: 404 }); + const body = asset.sha256[0]!.repeat(3); + return new Response(body, { headers: { "Content-Length": String(body.length) } }); + }); +} + +function pointerTag(objects: Map): string | undefined { + const pointer = objects.get(STABLE); + return pointer ? parseMirrorPointer(JSON.parse(pointer.body)).handoff.tag : undefined; +} + +beforeEach(() => { + vi.stubGlobal( + "FixedLengthStream", + class extends TransformStream { + constructor(_length: number) { + super(); + } + }, + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("desktop release mirror", () => { + it("copies every installer with its digest, then promotes the Stable pointer last", async () => { + const { bucket, objects, writes, put } = memoryBucket(); + vi.stubGlobal("fetch", githubFetch()); + + const result = await syncDesktopMirror(bucket, NOW); + + expect(result).toEqual({ + tag: "v0.6.28", + copied: [ + "Scient-0.6.28-arm64.dmg", + "Scient-0.6.28-x64.dmg", + "Scient-0.6.28-x64.exe", + "Scient-0.6.28-x86_64.AppImage", + ], + promoted: true, + superseded: false, + }); + expect(writes.at(-1)).toBe(STABLE); + expect(writes).not.toContain("scient-desktop/v0.6.28/latest.yml"); + expect(put).toHaveBeenCalledWith( + "scient-desktop/v0.6.28/Scient-0.6.28-x64.exe", + expect.anything(), + expect.objectContaining({ + sha256: "3".repeat(64), + customMetadata: { sha256: "3".repeat(64) }, + httpMetadata: expect.objectContaining({ + contentDisposition: 'attachment; filename="Scient-0.6.28-x64.exe"', + }), + }), + ); + expect(JSON.parse(objects.get(STABLE)!.body)).toEqual({ + schemaVersion: 1, + handoff, + publishedAt: "2026-11-10T16:14:08.000Z", + mirroredAt: NOW.toISOString(), + checkedAt: NOW.toISOString(), + }); + }); + + it("only refreshes the check time when the current release is already mirrored", async () => { + const first = memoryBucket(); + vi.stubGlobal("fetch", githubFetch()); + await syncDesktopMirror(first.bucket, new Date("2026-11-13T11:00:00.000Z")); + const fetchMock = githubFetch(); + vi.stubGlobal("fetch", fetchMock); + + const result = await syncDesktopMirror(first.bucket, NOW); + + expect(result).toEqual({ tag: "v0.6.28", copied: [], promoted: false, superseded: false }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(JSON.parse(first.objects.get(STABLE)!.body)).toMatchObject({ + mirroredAt: "2026-11-13T11:00:00.000Z", + checkedAt: NOW.toISOString(), + }); + }); + + it("never replaces a published installer whose digest the handoff later changes", async () => { + const { bucket, objects } = memoryBucket(); + vi.stubGlobal("fetch", githubFetch()); + await syncDesktopMirror(bucket, NOW); + const pointerBefore = objects.get(STABLE)!.body; + vi.stubGlobal( + "fetch", + githubFetch(() => handoffFor("0.6.28", "9234")), + ); + + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/never replaced/); + expect(objects.get("scient-desktop/v0.6.28/Scient-0.6.28-arm64.dmg")?.body).toBe("111"); + expect(objects.get(STABLE)!.body).toBe(pointerBefore); + }); + + it("keeps a newer promotion when an older, slower run finishes later", async () => { + const { bucket, objects } = memoryBucket(); + let latest = handoffFor("0.6.28"); + let releaseSlowCopy!: () => void; + const slowCopy = new Promise((resolve) => { + releaseSlowCopy = resolve; + }); + vi.stubGlobal( + "fetch", + githubFetch(() => latest, { + "https://github.com/ScientFactory/scient-desktop/releases/download/v0.6.28/Scient-0.6.28-x64.exe": + async () => { + await slowCopy; + return new Response("333", { headers: { "Content-Length": "3" } }); + }, + }), + ); + + const olderRun = syncDesktopMirror(bucket, NOW); + await vi.waitFor(() => + expect(objects.has("scient-desktop/v0.6.28/Scient-0.6.28-x64.dmg")).toBe(true), + ); + latest = handoffFor("0.6.29"); + await expect(syncDesktopMirror(bucket, NOW)).resolves.toMatchObject({ promoted: true }); + releaseSlowCopy(); + + await expect(olderRun).resolves.toMatchObject({ + tag: "v0.6.28", + promoted: false, + superseded: true, + }); + expect(pointerTag(objects)).toBe("v0.6.29"); + }); + + it("keeps a newer promotion when an older run's GitHub answer arrives late", async () => { + const { bucket, objects } = memoryBucket(); + vi.stubGlobal( + "fetch", + githubFetch(() => handoffFor("0.6.28")), + ); + await syncDesktopMirror(bucket, NOW); + let releaseOldAnswer!: () => void; + const oldAnswer = new Promise((resolve) => { + releaseOldAnswer = resolve; + }); + let handoffRequests = 0; + vi.stubGlobal( + "fetch", + githubFetch(() => handoffFor("0.6.29"), { + [HANDOFF_URL]: async () => { + handoffRequests += 1; + // The first request is answered late, with the release that was latest when it began. + if (handoffRequests === 1) { + await oldAnswer; + return Response.json(handoffFor("0.6.28")); + } + return Response.json(handoffFor("0.6.29")); + }, + }), + ); + + const olderRun = syncDesktopMirror(bucket, NOW); + await vi.waitFor(() => expect(handoffRequests).toBe(1)); + await expect(syncDesktopMirror(bucket, NOW)).resolves.toMatchObject({ + tag: "v0.6.29", + promoted: true, + }); + releaseOldAnswer(); + + await expect(olderRun).resolves.toMatchObject({ tag: "v0.6.28", superseded: true }); + expect(pointerTag(objects)).toBe("v0.6.29"); + }); + + it("leaves the previous pointer in place when an installer does not match the handoff", async () => { + const previous = JSON.stringify({ schemaVersion: 1, marker: "previous" }); + const { bucket, objects } = memoryBucket({ [STABLE]: previous }); + vi.stubGlobal( + "fetch", + githubFetch(undefined, { + "https://github.com/ScientFactory/scient-desktop/releases/download/v0.6.28/Scient-0.6.28-x64.dmg": + () => new Response("999", { headers: { "Content-Length": "3" } }), + }), + ); + + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/SHA-256/); + expect(objects.get(STABLE)?.body).toBe(previous); + expect(objects.has("scient-desktop/v0.6.28/Scient-0.6.28-x64.dmg")).toBe(false); + }); + + it("refuses an installer whose declared size differs from the handoff", async () => { + const { bucket, objects } = memoryBucket(); + vi.stubGlobal( + "fetch", + githubFetch(undefined, { + "https://github.com/ScientFactory/scient-desktop/releases/download/v0.6.28/Scient-0.6.28-arm64.dmg": + () => new Response("1111", { headers: { "Content-Length": "4" } }), + }), + ); + + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/4 bytes/); + expect(objects.size).toBe(0); + }); + + it("does nothing when GitHub's handoff is unavailable", async () => { + const { bucket, writes } = memoryBucket(); + vi.stubGlobal( + "fetch", + githubFetch(undefined, { [HANDOFF_URL]: () => new Response(null, { status: 429 }) }), + ); + + await expect(syncDesktopMirror(bucket, NOW)).rejects.toThrow(/429/); + expect(writes).toEqual([]); + }); +}); diff --git a/workers/release-mirror/src/mirror.ts b/workers/release-mirror/src/mirror.ts new file mode 100644 index 0000000..5af0ce1 --- /dev/null +++ b/workers/release-mirror/src/mirror.ts @@ -0,0 +1,159 @@ +// FILE: mirror.ts +// Purpose: Copies the current Stable desktop installers from GitHub into ScientFactory's R2 bucket. +// Layer: Cloudflare Worker (scheduled) + +import { readHandoff } from "../../../functions/_lib/latest-release"; +import { readMirroredRelease, writeMirrorPointer } from "../../../functions/_lib/release-mirror"; +import { + githubReleaseDownloadUrl, + parseReleaseHandoff, + releaseAssetContentType, + requiredInstallerNames, + type ReleaseHandoffAsset, +} from "../../../src/lib/release-handoff"; +import { + DESKTOP_MIRROR_STABLE_KEY, + desktopMirrorKey, + GITHUB_RELEASE_HANDOFF_URL, +} from "../../../src/lib/release-source"; + +const USER_AGENT = "ScientFactory-release-mirror"; +const HANDOFF_TIMEOUT_MS = 30_000; +const INSTALLER_TIMEOUT_MS = 10 * 60_000; + +export interface MirrorSyncResult { + readonly tag: string; + /** Installers copied this run; empty when the bucket already held every verified copy. */ + readonly copied: readonly string[]; + /** True when this run moved the Stable pointer to a different release. */ + readonly promoted: boolean; + /** True when another run wrote the pointer first, so this run left it alone. */ + readonly superseded: boolean; +} + +async function fetchLatestHandoff(): Promise<{ raw: unknown; publishedAt: string | null }> { + const response = await fetch(GITHUB_RELEASE_HANDOFF_URL, { + headers: { Accept: "application/octet-stream, application/json", "User-Agent": USER_AGENT }, + signal: AbortSignal.timeout(HANDOFF_TIMEOUT_MS), + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`GitHub returned ${response.status} for the release handoff.`); + } + const raw = await readHandoff(response); + const lastModified = Date.parse(response.headers.get("Last-Modified") ?? ""); + return { + raw, + publishedAt: Number.isFinite(lastModified) ? new Date(lastModified).toISOString() : null, + }; +} + +/** + * Copies one installer unless the bucket already holds it with the handoff's digest. + * Installer keys are immutable: bytes already published under a tag are never replaced, since + * the current pointer and year-long caches may still serve them. A changed digest needs a new tag. + */ +async function mirrorInstaller( + bucket: R2Bucket, + tag: string, + asset: ReleaseHandoffAsset, +): Promise { + const key = desktopMirrorKey(tag, asset.name); + const verifyExisting = (existing: R2Object | null): false => { + if (existing?.size === asset.size && existing.customMetadata?.sha256 === asset.sha256) { + return false; + } + throw new Error( + `${key} already holds other bytes (sha256 ${existing?.customMetadata?.sha256 ?? "unrecorded"}); ` + + `the handoff now says ${asset.sha256}. Published installers are never replaced.`, + ); + }; + const existing = await bucket.head(key); + if (existing !== null) return verifyExisting(existing); + + const response = await fetch(githubReleaseDownloadUrl(tag, asset.name), { + headers: { "User-Agent": USER_AGENT }, + signal: AbortSignal.timeout(INSTALLER_TIMEOUT_MS), + }); + if (!response.ok || response.body === null) { + await response.body?.cancel(); + throw new Error(`GitHub returned ${response.status} for ${asset.name}.`); + } + const declared = response.headers.get("Content-Length"); + if (declared !== null && Number(declared) !== asset.size) { + await response.body.cancel(); + throw new Error(`${asset.name} is ${declared} bytes; the handoff says ${asset.size}.`); + } + + // R2 needs a known length for a streamed body, and rejects the write when the bytes do not + // hash to `sha256`. `If-None-Match: *` keeps a concurrent run from replacing the object. + const sized = new FixedLengthStream(asset.size); + const stopPiping = new AbortController(); + const piping = response.body.pipeTo(sized.writable, { signal: stopPiping.signal }); + let written: R2Object | null; + try { + written = await bucket.put(key, sized.readable, { + onlyIf: new Headers({ "If-None-Match": "*" }), + sha256: asset.sha256, + httpMetadata: { + contentType: releaseAssetContentType(asset.name), + contentDisposition: `attachment; filename="${asset.name}"`, + cacheControl: "public, max-age=31536000, immutable", + }, + customMetadata: { sha256: asset.sha256 }, + }); + } catch (error) { + stopPiping.abort(); + await piping.catch(() => undefined); + throw error; + } + if (written === null) { + stopPiping.abort(); + await piping.catch(() => undefined); + return verifyExisting(await bucket.head(key)); + } + await piping; + return true; +} + +/** + * Brings the bucket up to GitHub's latest Stable release. Installers are copied (or confirmed) + * first; the Stable pointer is written last, so it never names bytes that are not there. + */ +export async function syncDesktopMirror(bucket: R2Bucket, now: Date): Promise { + // The pointer version this run may replace, captured before anything this run learns from + // GitHub: a run whose GitHub answer predates another run's promotion must lose its swap. + const observed = await bucket.head(DESKTOP_MIRROR_STABLE_KEY); + const { raw, publishedAt } = await fetchLatestHandoff(); + const handoff = parseReleaseHandoff(raw); + // An unreadable pointer is replaced below rather than blocking the mirror. + const current = await readMirroredRelease(bucket).catch(() => null); + + const copied: string[] = []; + for (const name of requiredInstallerNames(handoff.version)) { + const asset = handoff.assets.find((candidate) => candidate.name === name); + if (!asset) throw new Error(`Handoff ${handoff.tag} has no ${name}.`); + if (await mirrorInstaller(bucket, handoff.tag, asset)) copied.push(name); + } + + const sameRelease = + current !== null && JSON.stringify(current.handoff) === JSON.stringify(handoff); + const at = now.toISOString(); + const written = await writeMirrorPointer( + bucket, + { + schemaVersion: 1, + handoff: raw, + publishedAt, + mirroredAt: sameRelease && copied.length === 0 ? current.pointer.mirroredAt : at, + checkedAt: at, + }, + observed?.etag ?? null, + ); + return { + tag: handoff.tag, + copied, + promoted: written && !sameRelease, + superseded: !written, + }; +} diff --git a/workers/release-mirror/tsconfig.json b/workers/release-mirror/tsconfig.json new file mode 100644 index 0000000..30ee76d --- /dev/null +++ b/workers/release-mirror/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2023"], + "types": ["@cloudflare/workers-types"], + "noEmit": true + }, + "include": ["./src/**/*.ts", "./worker-configuration.d.ts"] +} diff --git a/workers/release-mirror/worker-configuration.d.ts b/workers/release-mirror/worker-configuration.d.ts new file mode 100644 index 0000000..9bbe1ef --- /dev/null +++ b/workers/release-mirror/worker-configuration.d.ts @@ -0,0 +1,36 @@ +/* eslint-disable */ +// Generated by Wrangler by running `wrangler types --config=workers/release-mirror/wrangler.jsonc --include-runtime=false workers/release-mirror/worker-configuration.d.ts` (hash: c584d57df5a14e3780f446a28d1637f3) +interface __BaseEnv_Env { + DESKTOP_DOWNLOADS: R2Bucket; +} +declare namespace Cloudflare { + interface GlobalProps { + mainModule: typeof import("./src/index"); + } + interface Env extends __BaseEnv_Env {} +} +interface Env extends __BaseEnv_Env {} +declare module "*.txt" { + const value: string; + export default value; +} +declare module "*.html" { + const value: string; + export default value; +} +declare module "*.sql" { + const value: string; + export default value; +} +declare module "*.bin" { + const value: ArrayBuffer; + export default value; +} +declare module "*.wasm" { + const value: WebAssembly.Module; + export default value; +} +declare module "*.wasm?module" { + const value: WebAssembly.Module; + export default value; +} diff --git a/workers/release-mirror/wrangler.jsonc b/workers/release-mirror/wrangler.jsonc new file mode 100644 index 0000000..07dbfd2 --- /dev/null +++ b/workers/release-mirror/wrangler.jsonc @@ -0,0 +1,20 @@ +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "scientfactory-release-mirror", + "main": "src/index.ts", + "compatibility_date": "2026-07-20", + "workers_dev": false, + "preview_urls": false, + "triggers": { + "crons": ["*/10 * * * *"], + }, + "r2_buckets": [ + { + "binding": "DESKTOP_DOWNLOADS", + "bucket_name": "scient-downloads", + }, + ], + "observability": { + "enabled": true, + }, +} diff --git a/wrangler.jsonc b/wrangler.jsonc index c120a60..f9e0524 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -12,6 +12,12 @@ "migrations_dir": "migrations", }, ], + "r2_buckets": [ + { + "binding": "DESKTOP_DOWNLOADS", + "bucket_name": "scient-downloads", + }, + ], "vars": { "IDENTITY_ANALYTICS_ENABLED": "true" }, "observability": { "enabled": true,