From 9f3bcf380eb898265953c342de8a74c4474b595c Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Fri, 18 Sep 2026 22:46:05 +0200
Subject: [PATCH 01/12] fix(releases): serve staged releases before any release
 reaches 100% (#78)

A prefix with no release at 100% made the default lookup throw a 500
before eligibility was checked, so no device got the staged build. That
is every device for a new prefix until its first release is fully
rolled out, and every JetKVM device the day no app or system row sits
at 100%.

The default lookup now returns null when nothing is at 100%. A device
inside the rollout bucket gets the staged release as before; a device
outside it gets a 404 saying no release is rolled out yet for its SKU,
instead of a 500.

Found by Bugbot on the release PR (#77).
---
 src/releases.ts       | 18 +++++++++++++-----
 test/releases.test.ts | 37 +++++++++++++++++++++++++++++++------
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/src/releases.ts b/src/releases.ts
index d34ba83..fb0293f 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -471,16 +471,18 @@ function dbReleaseToMetadata(release: DbRelease, sku: string): ReleaseMetadata {
   };
 }
 
-async function getDefaultRelease(prefix: string, sku: string): Promise<DbRelease> {
+/**
+ * Newest fully rolled out release for the prefix, or null when no release
+ * has reached 100% yet (a prefix whose first release is still staged).
+ */
+async function getDefaultRelease(prefix: string, sku: string): Promise<DbRelease | null> {
   const rolledOutReleases = await prisma.release.findMany({
     where: { type: prefix, rolloutPercentage: 100 },
     select: compatibleReleaseSelect(sku),
   });
 
   if (rolledOutReleases.length === 0) {
-    throw new InternalServerError(
-      `No default release found for type ${prefix} and SKU "${sku}"`,
-    );
+    return null;
   }
 
   // Only consider releases that ship a binary for this SKU. Without this,
@@ -605,7 +607,13 @@ export async function Retrieve(req: Request, res: Response) {
         latest.artifacts.length > 0 &&
         (await isDeviceEligibleForLatestRelease(latest.rolloutPercentage, deviceId));
 
-      offered[kind] = dbReleaseToMetadata(useLatest ? latest : fallback, sku);
+      const chosen = useLatest ? latest : fallback;
+      if (!chosen) {
+        throw new NotFoundError(
+          `No ${prefix} release is rolled out yet for SKU "${sku}"`,
+        );
+      }
+      offered[kind] = dbReleaseToMetadata(chosen, sku);
     }),
   );
 
diff --git a/test/releases.test.ts b/test/releases.test.ts
index cc32725..9d672bc 100644
--- a/test/releases.test.ts
+++ b/test/releases.test.ts
@@ -587,15 +587,26 @@ describe("Retrieve handler", () => {
       expect(s3Mock.commandCalls(GetObjectCommand)).toHaveLength(0);
     });
 
-    it("fails when no fully rolled out default exists for background checks", async () => {
+    it("serves a staged release to an in-bucket device even when nothing is at 100%", async () => {
+      // early-adopter hashes to rollout bucket 8, inside a 50% rollout.
+      await testPrisma.release.updateMany({ data: { rolloutPercentage: 50 } });
+
+      const res = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "early-adopter" }), res);
+
+      expect(jsonBody(res)).toMatchObject({ appVersion: "1.2.0", systemVersion: "1.2.0" });
+    });
+
+    it("answers 404, not 500, to an out-of-bucket device when nothing is at 100%", async () => {
+      // late-adopter hashes to rollout bucket 95, outside a 50% rollout.
       await testPrisma.release.updateMany({ data: { rolloutPercentage: 50 } });
 
       await expect(
-        Retrieve(
-          createMockRequest({ deviceId: "no-default-device" }),
-          createMockResponse(),
-        ),
-      ).rejects.toThrow(InternalServerError);
+        Retrieve(createMockRequest({ deviceId: "late-adopter" }), createMockResponse()),
+      ).rejects.toThrow(NotFoundError);
+      await expect(
+        Retrieve(createMockRequest({ deviceId: "late-adopter" }), createMockResponse()),
+      ).rejects.toThrow(/No (app|system) release is rolled out yet for SKU "jetkvm-v2"/);
     });
   });
 
@@ -789,6 +800,20 @@ describe("Retrieve handler", () => {
       ).rejects.toThrow('Version 1.0.0 has no artifact for SKU "jetkvm-mini-ethernet"');
     });
 
+    it("serves the first staged mini release to an in-bucket device and 404s the rest", async () => {
+      // No mini row at 100% yet: the first release is at 10%.
+      // early-adopter hashes to bucket 8, late-adopter to bucket 95.
+      await createDbRelease("mini", "1.0.0", 10, miniArtifacts("1.0.0"));
+
+      const inBucket = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "early-adopter", sku: MINI_ETHERNET_SKU }), inBucket);
+      expect(jsonBody(inBucket)).toMatchObject({ systemVersion: "1.0.0" });
+
+      await expect(
+        Retrieve(createMockRequest({ deviceId: "late-adopter", sku: MINI_ETHERNET_SKU }), createMockResponse()),
+      ).rejects.toThrow('No mini release is rolled out yet for SKU "jetkvm-mini-ethernet"');
+    });
+
     it("fails when no mini release exists for the SKU", async () => {
       const res = createMockResponse();
       await expect(

From a01f7a42f1dc0fe90b11e5ebbe784d9e263192dc Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Sat, 19 Sep 2026 13:24:23 +0200
Subject: [PATCH 02/12] Releases: sync new R2 releases from the API every 30
 minutes (#81)

* feat(releases): sync new R2 releases from the API every 30 minutes

The release sync only ran when an operator invoked scripts/sync-releases.ts
by hand, so a firmware upload sat in R2 until someone remembered to run it.
The API process now runs the same sync on start and every 30 minutes,
registering each new stable version at the default 10% rollout.

The non-interactive core (bucket listing, artifact collection, DB insert)
moves from the script into src/release-sync.ts so the API can import it;
the script keeps the GPG check and the confirmation prompt and passes them
in as the release decider. Both entry points share one R2 client via src/s3.ts.

Scheduled runs skip a tick while the previous run is still going, log and
survive a failed run, and treat a unique-constraint hit on insert as
"already synced" so several API instances can race without an error.
The existence check now precedes the R2 artifact walk, so a tick over an
already-synced bucket costs one list call per prefix plus one DB lookup per
version.

* fix(release-sync): defer versions whose upload is still settling

A scheduled tick can land while a version is still being uploaded and
register it with a partial SKU set or a hash that changes afterwards. Sync
never rewrites a row, so that snapshot would be permanent.

Before collecting artifacts, list every object under the version folder
and skip the version when the newest object changed in the last 10
minutes. The window is shorter than the schedule interval, so a real
upload costs at most one extra tick.

* refactor(release-sync): share R2 config, paginate via SDK, probe SKUs concurrently

* src/s3.ts now exports bucketName and baseUrl next to the client, so the
  request handlers, the scheduler and the script read R2 config from one
  place instead of three.
* The upload settle window moves into SyncConfig.uploadSettleMs. The
  scheduler sets it; the one-shot operator script leaves it unset because an
  operator sees the artifact list before confirming and has no next run.
* newestUploadTime uses the SDK's paginateListObjectsV2 instead of a
  hand-rolled continuation loop.
* collectReleaseArtifacts probes SKUs concurrently and folds the results in
  SKU order, so artifact order and the primary artifact are unchanged.
* Tests drop empty per-prefix listing stubs that the file-level default
  already covers.

* fix(release-sync): take the settle listing after the artifact scan

Listing the version folder before the scan left a gap: an upload that
started between the listing and the scan passed the check and was captured
partially. Listing after the scan closes it, because any upload active
during the scan leaves an object newer than the window and the snapshot
is discarded instead of registered.

* fix(release-sync): paginate the version listing

ListObjectsV2 returns at most 1,000 common prefixes per page. Once a
release prefix grows past that, versions on later pages were never seen
on any tick. listStableVersions now walks every page with the SDK
paginator, as newestUploadTime already does, and the mock in the sync
tests can serve a truncated listing.
---
 scripts/sync-releases.ts   | 411 ++++++-------------------------------
 src/index.ts               |   5 +
 src/release-sync.ts        | 390 +++++++++++++++++++++++++++++++++++
 src/releases.ts            |  68 +-----
 src/s3.ts                  |  57 +++++
 test/sync-releases.test.ts | 289 ++++++++++++++++++++++++--
 6 files changed, 794 insertions(+), 426 deletions(-)
 create mode 100644 src/release-sync.ts
 create mode 100644 src/s3.ts

diff --git a/scripts/sync-releases.ts b/scripts/sync-releases.ts
index c1fa91f..6a2edb4 100644
--- a/scripts/sync-releases.ts
+++ b/scripts/sync-releases.ts
@@ -8,54 +8,28 @@ import { Readable } from "node:stream";
 import { pipeline } from "node:stream/promises";
 import { createInterface } from "node:readline/promises";
 
-import {
-  GetObjectCommand,
-  HeadObjectCommand,
-  ListObjectsV2Command,
-  S3Client,
-} from "@aws-sdk/client-s3";
+import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import { PrismaClient } from "@prisma/client";
 import semver from "semver";
 
-import { objectKeyFromArtifactUrl, streamToString } from "../src/helpers";
-import { OTA_PREFIXES, legacyCompatibleSkus, otaFileForPrefix, skusForPrefix } from "../src/skus";
-
-/** An R2 prefix, which is also the Release.type column value. */
-type ReleaseType = string;
+import { objectKeyFromArtifactUrl } from "../src/helpers";
+import { baseUrl, bucketName, s3Client, s3ObjectExists } from "../src/s3";
+import {
+  DEFAULT_ROLLOUT_PERCENTAGE,
+  createAtDefaultRollout,
+  syncReleases,
+  type ReleaseArtifactInput,
+  type ReleaseDecider,
+  type ReleaseType,
+  type SyncClients,
+  type SyncConfig,
+} from "../src/release-sync";
+
+// Operator front end for src/release-sync.ts: signature verification output
+// and a confirmation prompt before each production DB write.
 
 const OTA_ROOT_KEY_FPR = "AF5A36A993D828FEFE7C18C2D1B9856C26A79E95";
 
-interface SyncClients {
-  prisma: PrismaClient;
-  s3Client: S3Client;
-}
-
-interface SyncConfig {
-  bucketName: string;
-  baseUrl: string;
-  skus?: string[];
-}
-
-interface ReleaseArtifactInput {
-  url: string;
-  hash: string;
-  compatibleSkus: string[];
-}
-
-const DEFAULT_ROLLOUT_PERCENTAGE = 10;
-
-type ReleaseOutcome =
-  | "created"
-  | "already-synced"
-  | "no-artifacts"
-  | "skipped"
-  | "aborted";
-
-type ReleaseDecision =
-  | { kind: "create"; rolloutPercentage: number }
-  | { kind: "skip" }
-  | { kind: "abort" };
-
 interface LatestExistingRelease {
   version: string;
   rolloutPercentage: number;
@@ -360,305 +334,41 @@ async function promptRolloutPercentage(
   }
 }
 
-async function confirmProductionCreate(
+function confirmProductionCreate(
   clients: SyncClients,
   config: SyncConfig,
-  type: ReleaseType,
-  version: string,
-  artifacts: ReleaseArtifactInput[],
-): Promise<ReleaseDecision> {
-  if (process.env.NODE_ENV !== "production") {
-    return { kind: "create", rolloutPercentage: DEFAULT_ROLLOUT_PERCENTAGE };
-  }
-
-  if (!stdin.isTTY || !stdout.isTTY) {
-    throw new Error(
-      "Production release sync requires an interactive terminal for DB write confirmation.",
-    );
-  }
-
-  const [artifactInfos, latestExisting] = await Promise.all([
-    loadArtifactDisplayInfo(clients, config, artifacts),
-    findLatestExistingRelease(clients.prisma, type),
-  ]);
+): ReleaseDecider {
+  return async (type, version, artifacts) => {
+    const [artifactInfos, latestExisting] = await Promise.all([
+      loadArtifactDisplayInfo(clients, config, artifacts),
+      findLatestExistingRelease(clients.prisma, type),
+    ]);
 
-  printArtifactSummary(type, version, artifactInfos, latestExisting);
+    printArtifactSummary(type, version, artifactInfos, latestExisting);
 
-  const readline = createInterface({ input: stdin, output: stdout });
-  try {
-    const rolloutPercentage = await promptRolloutPercentage(readline);
+    const readline = createInterface({ input: stdin, output: stdout });
+    try {
+      const rolloutPercentage = await promptRolloutPercentage(readline);
 
-    const confirmation = (
-      await readline.question(
-        `  Create production ${type} release ${version} at ${rolloutPercentage}% rollout? [y/N/a (abort run)] `,
+      const confirmation = (
+        await readline.question(
+          `  Create production ${type} release ${version} at ${rolloutPercentage}% rollout? [y/N/a (abort run)] `,
+        )
       )
-    )
-      .trim()
-      .toLowerCase();
-
-    if (["a", "abort"].includes(confirmation)) {
-      return { kind: "abort" };
-    }
-    if (!["y", "yes"].includes(confirmation)) {
-      return { kind: "skip" };
-    }
-    return { kind: "create", rolloutPercentage };
-  } finally {
-    readline.close();
-  }
-}
-
-function isS3NotFound(error: any): boolean {
-  return (
-    error.name === "NotFound" ||
-    error.name === "NoSuchKey" ||
-    error.$metadata?.httpStatusCode === 404
-  );
-}
-
-async function s3ObjectExists(
-  s3Client: S3Client,
-  bucketName: string,
-  key: string,
-): Promise<boolean> {
-  try {
-    await s3Client.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }));
-    return true;
-  } catch (error: any) {
-    if (isS3NotFound(error)) {
-      return false;
-    }
-    throw error;
-  }
-}
-
-async function versionHasSkuSupport(
-  s3Client: S3Client,
-  bucketName: string,
-  type: ReleaseType,
-  version: string,
-): Promise<boolean> {
-  const response = await s3Client.send(
-    new ListObjectsV2Command({
-      Bucket: bucketName,
-      Prefix: `${type}/${version}/skus/`,
-      MaxKeys: 1,
-    }),
-  );
-  return (response.Contents?.length ?? 0) > 0;
-}
-
-async function readHash(
-  s3Client: S3Client,
-  bucketName: string,
-  artifactPath: string,
-): Promise<string | undefined> {
-  try {
-    const response = await s3Client.send(
-      new GetObjectCommand({
-        Bucket: bucketName,
-        Key: `${artifactPath}.sha256`,
-      }),
-    );
-    return streamToString(response.Body);
-  } catch (error: any) {
-    if (isS3NotFound(error)) {
-      return undefined;
-    }
-    throw error;
-  }
-}
-
-function addArtifact(
-  artifactsByUrl: Map<string, ReleaseArtifactInput>,
-  url: string,
-  hash: string,
-  sku: string,
-): void {
-  const artifact = artifactsByUrl.get(url);
-  if (artifact) {
-    if (!artifact.compatibleSkus.includes(sku)) {
-      artifact.compatibleSkus.push(sku);
-    }
-    return;
-  }
-
-  artifactsByUrl.set(url, { url, hash, compatibleSkus: [sku] });
-}
-
-export async function collectReleaseArtifacts(
-  clients: Pick<SyncClients, "s3Client">,
-  config: SyncConfig,
-  type: ReleaseType,
-  version: string,
-): Promise<ReleaseArtifactInput[]> {
-  const skus = config.skus ?? skusForPrefix(type);
-  const artifactFileName = otaFileForPrefix(type);
-
-  if (!(await versionHasSkuSupport(clients.s3Client, config.bucketName, type, version))) {
-    // Pre-SKU artifacts (no skus/ folder) are only safe on the SKUs that
-    // predate the layout. A type with no legacy form treats a version
-    // without skus/ as an upload mistake, not a release.
-    const compatibleSkus = legacyCompatibleSkus(type);
-    if (compatibleSkus.length === 0) {
-      return [];
-    }
-
-    const artifactPath = `${type}/${version}/${artifactFileName}`;
-    const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
-    if (!hash) {
-      return [];
-    }
-
-    return [
-      {
-        url: `${config.baseUrl}/${artifactPath}`,
-        hash,
-        compatibleSkus,
-      },
-    ];
-  }
-
-  const artifactsByUrl = new Map<string, ReleaseArtifactInput>();
-  for (const sku of skus) {
-    const artifactPath = `${type}/${version}/skus/${sku}/${artifactFileName}`;
-    if (!(await s3ObjectExists(clients.s3Client, config.bucketName, artifactPath))) {
-      continue;
-    }
-
-    const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
-    if (!hash) {
-      continue;
-    }
-    addArtifact(artifactsByUrl, `${config.baseUrl}/${artifactPath}`, hash, sku);
-  }
-
-  return Array.from(artifactsByUrl.values());
-}
-
-async function listStableVersions(
-  s3Client: S3Client,
-  bucketName: string,
-  type: ReleaseType,
-): Promise<string[]> {
-  const response = await s3Client.send(
-    new ListObjectsV2Command({
-      Bucket: bucketName,
-      Prefix: `${type}/`,
-      Delimiter: "/",
-    }),
-  );
-
-  return (response.CommonPrefixes ?? [])
-    .map(cp => cp.Prefix?.split("/")[1])
-    .filter((version): version is string => Boolean(version))
-    .filter(
-      version => Boolean(semver.valid(version)) && semver.prerelease(version) === null,
-    )
-    .sort(semver.compare);
-}
-
-async function syncRelease(
-  clients: SyncClients,
-  config: SyncConfig,
-  type: ReleaseType,
-  version: string,
-  artifacts: ReleaseArtifactInput[],
-): Promise<ReleaseOutcome> {
-  if (artifacts.length === 0) {
-    console.log(`[sync-releases] ${type} ${version}: skipped, no compatible artifacts`);
-    return "no-artifacts";
-  }
-
-  // Sync only registers brand-new releases. Existing rows (rollout state, URLs,
-  // artifact compatibility) are left untouched — backfills/repairs are handled
-  // by one-off scripts so a routine sync run can never rewrite production data.
-  const existing = await clients.prisma.release.findUnique({
-    where: { version_type: { version, type } },
-    select: { id: true },
-  });
-
-  if (existing) {
-    console.log(`[sync-releases] ${type} ${version}: already synced, skipping`);
-    return "already-synced";
-  }
-
-  const decision = await confirmProductionCreate(
-    clients,
-    config,
-    type,
-    version,
-    artifacts,
-  );
-  if (decision.kind === "abort") {
-    console.log(`[sync-releases] ${type} ${version}: aborted by user`);
-    return "aborted";
-  }
-  if (decision.kind === "skip") {
-    console.log(`[sync-releases] ${type} ${version}: skipped by user`);
-    return "skipped";
-  }
-
-  const primaryArtifact = artifacts[0];
-  await clients.prisma.release.create({
-    data: {
-      version,
-      type,
-      rolloutPercentage: decision.rolloutPercentage,
-      url: primaryArtifact.url,
-      hash: primaryArtifact.hash,
-      artifacts: {
-        create: artifacts.map(artifact => ({
-          url: artifact.url,
-          hash: artifact.hash,
-          compatibleSkus: artifact.compatibleSkus,
-        })),
-      },
-    },
-  });
-
-  console.log(
-    `[sync-releases] ${type} ${version}: created with ${artifacts.length} artifact(s) at ${decision.rolloutPercentage}% rollout`,
-  );
-  return "created";
-}
-
-export async function syncReleases(
-  clients: SyncClients,
-  config: SyncConfig,
-): Promise<void> {
-  const stats: Record<ReleaseOutcome, number> = {
-    created: 0,
-    "already-synced": 0,
-    "no-artifacts": 0,
-    skipped: 0,
-    aborted: 0,
-  };
-  let abortedAt: { type: ReleaseType; version: string } | null = null;
-
-  outer: for (const type of OTA_PREFIXES) {
-    const versions = await listStableVersions(clients.s3Client, config.bucketName, type);
-
-    for (const version of versions) {
-      const artifacts = await collectReleaseArtifacts(clients, config, type, version);
-      const outcome = await syncRelease(clients, config, type, version, artifacts);
-      stats[outcome]++;
+        .trim()
+        .toLowerCase();
 
-      if (outcome === "aborted") {
-        abortedAt = { type, version };
-        break outer;
+      if (["a", "abort"].includes(confirmation)) {
+        return { kind: "abort" };
       }
+      if (!["y", "yes"].includes(confirmation)) {
+        return { kind: "skip" };
+      }
+      return { kind: "create", rolloutPercentage };
+    } finally {
+      readline.close();
     }
-  }
-
-  if (abortedAt) {
-    console.log(
-      `[sync-releases] aborted at ${abortedAt.type} ${abortedAt.version}; remaining versions in this run were not processed`,
-    );
-  }
-  console.log(
-    `[sync-releases] done: created=${stats.created} skipped-by-user=${stats.skipped} already-synced=${stats["already-synced"]} no-artifacts=${stats["no-artifacts"]}`,
-  );
+  };
 }
 
 function describeDbTarget(): string {
@@ -678,35 +388,32 @@ function describeDbTarget(): string {
 
 async function main(): Promise<void> {
   console.log(
-    `[sync-releases] env=${process.env.NODE_ENV ?? "(unset)"} db=${describeDbTarget()} bucket=${process.env.R2_BUCKET ?? "(unset)"}`,
+    `[sync-releases] env=${process.env.NODE_ENV ?? "(unset)"} db=${describeDbTarget()} bucket=${bucketName ?? "(unset)"}`,
   );
 
+  const isProduction = process.env.NODE_ENV === "production";
+  if (isProduction && (!stdin.isTTY || !stdout.isTTY)) {
+    throw new Error(
+      "Production release sync requires an interactive terminal for DB write confirmation.",
+    );
+  }
+
   const prisma = new PrismaClient();
-  const s3Client = new S3Client({
-    endpoint: process.env.R2_ENDPOINT!,
-    credentials: {
-      accessKeyId: process.env.R2_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
-    },
-    region: "auto",
-  });
+  const clients: SyncClients = { prisma, s3Client };
+  const config: SyncConfig = { bucketName, baseUrl };
 
   try {
     await syncReleases(
-      { prisma, s3Client },
-      {
-        bucketName: process.env.R2_BUCKET!,
-        baseUrl: process.env.R2_CDN_URL!,
-      },
+      clients,
+      config,
+      isProduction ? confirmProductionCreate(clients, config) : createAtDefaultRollout,
     );
   } finally {
     await prisma.$disconnect();
   }
 }
 
-if (require.main === module) {
-  main().catch(error => {
-    console.error("[sync-releases] failed", error);
-    process.exit(1);
-  });
-}
+main().catch(error => {
+  console.error("[sync-releases] failed", error);
+  process.exit(1);
+});
diff --git a/src/index.ts b/src/index.ts
index b6fdecb..2a235c1 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -13,6 +13,8 @@ import * as Releases from "./releases";
 import { HttpError } from "./errors";
 import { authenticated } from "./auth";
 import { prisma } from "./db";
+import { baseUrl, bucketName, s3Client } from "./s3";
+import { scheduleReleaseSync } from "./release-sync";
 import { initializeWebRTCSignaling } from "./webrtc-signaling";
 
 declare global {
@@ -218,3 +220,6 @@ const server = app.listen(PORT, () => {
 });
 
 initializeWebRTCSignaling(server);
+
+// Register new R2 releases at the default rollout, now and every 30 minutes.
+scheduleReleaseSync({ prisma, s3Client }, { bucketName, baseUrl });
diff --git a/src/release-sync.ts b/src/release-sync.ts
new file mode 100644
index 0000000..2911427
--- /dev/null
+++ b/src/release-sync.ts
@@ -0,0 +1,390 @@
+import { GetObjectCommand, S3Client, paginateListObjectsV2 } from "@aws-sdk/client-s3";
+import { Prisma, PrismaClient } from "@prisma/client";
+import semver from "semver";
+
+import { streamToString } from "./helpers";
+import { isS3NotFound, s3ObjectExists, versionHasSkuSupport } from "./s3";
+import { OTA_PREFIXES, legacyCompatibleSkus, otaFileForPrefix, skusForPrefix } from "./skus";
+
+/** An R2 prefix, which is also the Release.type column value. */
+export type ReleaseType = string;
+
+export interface SyncClients {
+  prisma: PrismaClient;
+  s3Client: S3Client;
+}
+
+export interface SyncConfig {
+  bucketName: string;
+  baseUrl: string;
+  skus?: string[];
+  /**
+   * Defer a version whose newest object changed within this many ms, checked
+   * after the artifact scan: the upload script writes several files per SKU,
+   * and sync never rewrites a row, so a scan that overlapped an upload would
+   * freeze a partial SKU set or a stale hash. Unset or 0 disables the check
+   * (an operator at a terminal can judge the artifact list for themselves).
+   */
+  uploadSettleMs?: number;
+}
+
+export interface ReleaseArtifactInput {
+  url: string;
+  hash: string;
+  compatibleSkus: string[];
+}
+
+export const DEFAULT_ROLLOUT_PERCENTAGE = 10;
+
+export type ReleaseOutcome =
+  | "created"
+  | "already-synced"
+  | "uploading"
+  | "no-artifacts"
+  | "skipped"
+  | "aborted";
+
+/** Settle window for unattended runs; shorter than the interval, so it costs at most one tick. */
+export const UPLOAD_SETTLE_MS = 10 * 60 * 1000;
+
+export type ReleaseDecision =
+  | { kind: "create"; rolloutPercentage: number }
+  | { kind: "skip" }
+  | { kind: "abort" };
+
+/**
+ * Decides whether a release that exists in R2 but not in the DB gets created.
+ * The interactive script asks the operator; the in-process scheduler always
+ * creates at the default rollout.
+ */
+export type ReleaseDecider = (
+  type: ReleaseType,
+  version: string,
+  artifacts: ReleaseArtifactInput[],
+) => Promise<ReleaseDecision>;
+
+export const createAtDefaultRollout: ReleaseDecider = async () => ({
+  kind: "create",
+  rolloutPercentage: DEFAULT_ROLLOUT_PERCENTAGE,
+});
+
+async function readHash(
+  s3Client: S3Client,
+  bucketName: string,
+  artifactPath: string,
+): Promise<string | undefined> {
+  try {
+    const response = await s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucketName,
+        Key: `${artifactPath}.sha256`,
+      }),
+    );
+    return streamToString(response.Body);
+  } catch (error: any) {
+    if (isS3NotFound(error)) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+function addArtifact(
+  artifactsByUrl: Map<string, ReleaseArtifactInput>,
+  url: string,
+  hash: string,
+  sku: string,
+): void {
+  const artifact = artifactsByUrl.get(url);
+  if (artifact) {
+    if (!artifact.compatibleSkus.includes(sku)) {
+      artifact.compatibleSkus.push(sku);
+    }
+    return;
+  }
+
+  artifactsByUrl.set(url, { url, hash, compatibleSkus: [sku] });
+}
+
+export async function collectReleaseArtifacts(
+  clients: Pick<SyncClients, "s3Client">,
+  config: SyncConfig,
+  type: ReleaseType,
+  version: string,
+): Promise<ReleaseArtifactInput[]> {
+  const skus = config.skus ?? skusForPrefix(type);
+  const artifactFileName = otaFileForPrefix(type);
+
+  if (!(await versionHasSkuSupport(clients.s3Client, config.bucketName, type, version))) {
+    // Pre-SKU artifacts (no skus/ folder) are only safe on the SKUs that
+    // predate the layout. A type with no legacy form treats a version
+    // without skus/ as an upload mistake, not a release.
+    const compatibleSkus = legacyCompatibleSkus(type);
+    if (compatibleSkus.length === 0) {
+      return [];
+    }
+
+    const artifactPath = `${type}/${version}/${artifactFileName}`;
+    const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
+    if (!hash) {
+      return [];
+    }
+
+    return [
+      {
+        url: `${config.baseUrl}/${artifactPath}`,
+        hash,
+        compatibleSkus,
+      },
+    ];
+  }
+
+  // SKUs are probed concurrently; folding afterwards in `skus` order keeps
+  // the primary artifact (artifacts[0]) and compatibleSkus order stable.
+  const found = await Promise.all(
+    skus.map(async sku => {
+      const artifactPath = `${type}/${version}/skus/${sku}/${artifactFileName}`;
+      if (!(await s3ObjectExists(clients.s3Client, config.bucketName, artifactPath))) {
+        return undefined;
+      }
+      const hash = await readHash(clients.s3Client, config.bucketName, artifactPath);
+      return hash ? { sku, url: `${config.baseUrl}/${artifactPath}`, hash } : undefined;
+    }),
+  );
+
+  const artifactsByUrl = new Map<string, ReleaseArtifactInput>();
+  for (const artifact of found) {
+    if (artifact) {
+      addArtifact(artifactsByUrl, artifact.url, artifact.hash, artifact.sku);
+    }
+  }
+  return Array.from(artifactsByUrl.values());
+}
+
+async function listStableVersions(
+  s3Client: S3Client,
+  bucketName: string,
+  type: ReleaseType,
+): Promise<string[]> {
+  const prefixes: string[] = [];
+  for await (const page of paginateListObjectsV2(
+    { client: s3Client },
+    { Bucket: bucketName, Prefix: `${type}/`, Delimiter: "/" },
+  )) {
+    for (const cp of page.CommonPrefixes ?? []) {
+      if (cp.Prefix) {
+        prefixes.push(cp.Prefix);
+      }
+    }
+  }
+
+  return prefixes
+    .map(prefix => prefix.split("/")[1])
+    .filter((version): version is string => Boolean(version))
+    .filter(
+      version => Boolean(semver.valid(version)) && semver.prerelease(version) === null,
+    )
+    .sort(semver.compare);
+}
+
+/** Newest LastModified among every object under `${type}/${version}/`, or undefined when empty. */
+async function newestUploadTime(
+  s3Client: S3Client,
+  bucketName: string,
+  type: ReleaseType,
+  version: string,
+): Promise<Date | undefined> {
+  let newest: Date | undefined;
+  for await (const page of paginateListObjectsV2(
+    { client: s3Client },
+    { Bucket: bucketName, Prefix: `${type}/${version}/` },
+  )) {
+    for (const object of page.Contents ?? []) {
+      if (object.LastModified && (!newest || object.LastModified > newest)) {
+        newest = object.LastModified;
+      }
+    }
+  }
+  return newest;
+}
+
+async function listSyncedVersions(prisma: PrismaClient, type: ReleaseType): Promise<Set<string>> {
+  const releases = await prisma.release.findMany({
+    where: { type },
+    select: { version: true },
+  });
+  return new Set(releases.map(release => release.version));
+}
+
+function isUniqueViolation(error: unknown): boolean {
+  return (
+    error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002"
+  );
+}
+
+async function createRelease(
+  clients: SyncClients,
+  config: SyncConfig,
+  decide: ReleaseDecider,
+  type: ReleaseType,
+  version: string,
+): Promise<ReleaseOutcome> {
+  const artifacts = await collectReleaseArtifacts(clients, config, type, version);
+
+  // Listed after the artifact scan on purpose: an upload that was active at
+  // any point during the scan leaves an object newer than the window, so the
+  // snapshot above is discarded rather than registered.
+  if (config.uploadSettleMs) {
+    const newest = await newestUploadTime(clients.s3Client, config.bucketName, type, version);
+    if (newest && Date.now() - newest.getTime() < config.uploadSettleMs) {
+      console.log(
+        `[sync-releases] ${type} ${version}: upload still settling (last object ${newest.toISOString()}), retrying next run`,
+      );
+      return "uploading";
+    }
+  }
+
+  if (artifacts.length === 0) {
+    console.log(`[sync-releases] ${type} ${version}: skipped, no compatible artifacts`);
+    return "no-artifacts";
+  }
+
+  const decision = await decide(type, version, artifacts);
+  if (decision.kind === "abort") {
+    console.log(`[sync-releases] ${type} ${version}: aborted by user`);
+    return "aborted";
+  }
+  if (decision.kind === "skip") {
+    console.log(`[sync-releases] ${type} ${version}: skipped by user`);
+    return "skipped";
+  }
+
+  const primaryArtifact = artifacts[0];
+  try {
+    await clients.prisma.release.create({
+      data: {
+        version,
+        type,
+        rolloutPercentage: decision.rolloutPercentage,
+        url: primaryArtifact.url,
+        hash: primaryArtifact.hash,
+        artifacts: {
+          create: artifacts.map(artifact => ({
+            url: artifact.url,
+            hash: artifact.hash,
+            compatibleSkus: artifact.compatibleSkus,
+          })),
+        },
+      },
+    });
+  } catch (error) {
+    // Another API instance can win the race between the version listing and
+    // this insert. The row it wrote is the one we wanted, so treat it as synced.
+    if (isUniqueViolation(error)) {
+      console.log(`[sync-releases] ${type} ${version}: created concurrently elsewhere, skipping`);
+      return "already-synced";
+    }
+    throw error;
+  }
+
+  console.log(
+    `[sync-releases] ${type} ${version}: created with ${artifacts.length} artifact(s) at ${decision.rolloutPercentage}% rollout`,
+  );
+  return "created";
+}
+
+/**
+ * Registers every stable version in R2 that has no Release row yet.
+ *
+ * Sync only registers brand-new releases. Existing rows (rollout state, URLs,
+ * artifact compatibility) are left untouched — backfills/repairs are handled
+ * by one-off scripts so a routine sync run can never rewrite production data.
+ * Known versions are loaded once per prefix, so a run over an already-synced
+ * bucket costs one R2 list and one DB query per prefix.
+ *
+ * Returns the per-outcome counts so callers can log or assert on them.
+ */
+export async function syncReleases(
+  clients: SyncClients,
+  config: SyncConfig,
+  decide: ReleaseDecider,
+): Promise<Record<ReleaseOutcome, number>> {
+  const stats: Record<ReleaseOutcome, number> = {
+    created: 0,
+    "already-synced": 0,
+    uploading: 0,
+    "no-artifacts": 0,
+    skipped: 0,
+    aborted: 0,
+  };
+  let abortedAt: { type: ReleaseType; version: string } | null = null;
+
+  outer: for (const type of OTA_PREFIXES) {
+    const [versions, synced] = await Promise.all([
+      listStableVersions(clients.s3Client, config.bucketName, type),
+      listSyncedVersions(clients.prisma, type),
+    ]);
+
+    for (const version of versions) {
+      const outcome = synced.has(version)
+        ? "already-synced"
+        : await createRelease(clients, config, decide, type, version);
+      stats[outcome]++;
+
+      if (outcome === "aborted") {
+        abortedAt = { type, version };
+        break outer;
+      }
+    }
+  }
+
+  if (abortedAt) {
+    console.log(
+      `[sync-releases] aborted at ${abortedAt.type} ${abortedAt.version}; remaining versions in this run were not processed`,
+    );
+  }
+  console.log(
+    `[sync-releases] done: created=${stats.created} skipped-by-user=${stats.skipped} already-synced=${stats["already-synced"]} uploading=${stats.uploading} no-artifacts=${stats["no-artifacts"]}`,
+  );
+  return stats;
+}
+
+const RELEASE_SYNC_INTERVAL_MS = 30 * 60 * 1000;
+
+/**
+ * Runs the sync now and then every `intervalMs`, inside the API process.
+ * A tick that fires while the previous run is still going is skipped, not
+ * queued. A failed run is logged and the schedule continues, so one bad R2
+ * or DB response never stops future syncs.
+ * Returns a function that stops the schedule.
+ */
+export function scheduleReleaseSync(
+  clients: SyncClients,
+  config: SyncConfig,
+  intervalMs: number = RELEASE_SYNC_INTERVAL_MS,
+): () => void {
+  let running = false;
+
+  const run = async () => {
+    if (running) {
+      console.warn("[sync-releases] previous run still in progress, skipping this tick");
+      return;
+    }
+    running = true;
+    try {
+      await syncReleases(
+        clients,
+        { uploadSettleMs: UPLOAD_SETTLE_MS, ...config },
+        createAtDefaultRollout,
+      );
+    } catch (error) {
+      console.error("[sync-releases] scheduled run failed", error);
+    } finally {
+      running = false;
+    }
+  };
+
+  void run();
+  const timer = setInterval(run, intervalMs);
+  return () => clearInterval(timer);
+}
diff --git a/src/releases.ts b/src/releases.ts
index fb0293f..33d5323 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -3,13 +3,9 @@ import { prisma } from "./db";
 import { BadRequestError, InternalServerError, NotFoundError } from "./errors";
 import semver from "semver";
 
-import {
-  GetObjectCommand,
-  HeadObjectCommand,
-  ListObjectsV2Command,
-  S3Client,
-} from "@aws-sdk/client-s3";
+import { GetObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
 import { LRUCache } from "lru-cache";
+import { baseUrl, bucketName, s3Client, s3ObjectExists, versionHasSkuSupport } from "./s3";
 
 import {
   getDeviceRolloutBucket,
@@ -101,15 +97,6 @@ interface DbRelease {
   }[];
 }
 
-const s3Client = new S3Client({
-  endpoint: process.env.R2_ENDPOINT!,
-  credentials: {
-    accessKeyId: process.env.R2_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
-  },
-  region: "auto",
-});
-
 const releaseCache = new LRUCache<string, ReleaseMetadata>({
   max: 1000,
   ttl: 5 * 60 * 1000, // 5 minutes
@@ -134,9 +121,6 @@ export function clearCaches() {
   sigUrlCache.clear();
 }
 
-const bucketName = process.env.R2_BUCKET;
-const baseUrl = process.env.R2_CDN_URL;
-
 /**
  * The one error for "this version ships no artifact for this SKU", whichever
  * path detects it: a skus/ folder without this SKU, a pre-SKU version asked
@@ -146,41 +130,7 @@ function noArtifactForSku(version: string, sku: string): NotFoundError {
   return new NotFoundError(`Version ${version} has no artifact for SKU "${sku}"`);
 }
 
-/**
- * Checks if an object exists in S3/R2 by attempting a HeadObjectCommand.
- * Returns true if the object exists, false otherwise.
- */
-async function s3ObjectExists(key: string): Promise<boolean> {
-  try {
-    await s3Client.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }));
-    return true;
-  } catch (error: any) {
-    // HeadObjectCommand throws NotFound, but some S3-compatible stores (like R2) may throw NoSuchKey
-    if (
-      error.name === "NotFound" ||
-      error.name === "NoSuchKey" ||
-      error.$metadata?.httpStatusCode === 404
-    ) {
-      return false;
-    }
-    throw error;
-  }
-}
-
-/**
- * Checks if a version was uploaded with SKU folder structure.
- * Returns true if any skus/ subfolder exists for this version.
- */
-async function versionHasSkuSupport(prefix: string, version: string): Promise<boolean> {
-  const response = await s3Client.send(
-    new ListObjectsV2Command({
-      Bucket: bucketName,
-      Prefix: `${prefix}/${version}/skus/`,
-      MaxKeys: 1,
-    }),
-  );
-  return (response.Contents?.length ?? 0) > 0;
-}
+const objectExists = (key: string) => s3ObjectExists(s3Client, bucketName, key);
 
 /**
  * Resolves the artifact path for a given version and SKU.
@@ -205,10 +155,10 @@ async function resolveArtifactPath(
   sku: string,
   file: string,
 ): Promise<string> {
-  if (await versionHasSkuSupport(prefix, version)) {
+  if (await versionHasSkuSupport(s3Client, bucketName, prefix, version)) {
     const skuPath = `${prefix}/${version}/skus/${sku}/${file}`;
 
-    if (await s3ObjectExists(skuPath)) {
+    if (await objectExists(skuPath)) {
       return skuPath;
     }
 
@@ -243,7 +193,7 @@ async function resolveSigUrl(
   try {
     const path = await resolveArtifactPath(artifact.prefix, version, sku, artifact.file);
     const sigKey = `${path}.sig`;
-    if (await s3ObjectExists(sigKey)) {
+    if (await objectExists(sigKey)) {
       const url = `${baseUrl}/${sigKey}`;
       sigUrlCache.set(cacheKey, url);
       return url;
@@ -389,7 +339,7 @@ async function resolveSigUrlFromArtifactUrl(
   const sigUrl = `${artifactUrl}.sig`;
   try {
     const sigKey = `${objectKeyFromArtifactUrl(artifactUrl)}.sig`;
-    if (await s3ObjectExists(sigKey)) {
+    if (await objectExists(sigKey)) {
       sigUrlCache.set(cacheKey, sigUrl);
       return sigUrl;
     }
@@ -693,7 +643,7 @@ export const RetrieveLatestSystemRecovery = cachedRedirect(
       recovery.file,
     );
 
-    if (!(await s3ObjectExists(artifactPath))) {
+    if (!(await objectExists(artifactPath))) {
       throw new NotFoundError(`Recovery image not found for version ${latestVersion}`);
     }
 
@@ -750,7 +700,7 @@ function latestArtifactRedirect(kind: OtaKind) {
         artifact.file,
       );
 
-      if (!(await s3ObjectExists(artifactPath))) {
+      if (!(await objectExists(artifactPath))) {
         throw new NotFoundError(`${prefix} artifact not found for version ${latestVersion}`);
       }
 
diff --git a/src/s3.ts b/src/s3.ts
new file mode 100644
index 0000000..1c702bb
--- /dev/null
+++ b/src/s3.ts
@@ -0,0 +1,57 @@
+import { HeadObjectCommand, ListObjectsV2Command, S3Client } from "@aws-sdk/client-s3";
+
+/** The one R2 client the API process shares between request handlers and background jobs. */
+export const s3Client = new S3Client({
+  endpoint: process.env.R2_ENDPOINT!,
+  credentials: {
+    accessKeyId: process.env.R2_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
+  },
+  region: "auto",
+});
+
+/** Bucket that holds release artifacts, and the public CDN origin they are served from. */
+export const bucketName = process.env.R2_BUCKET!;
+export const baseUrl = process.env.R2_CDN_URL!;
+
+/** HeadObject throws NotFound, but some S3-compatible stores (like R2) may throw NoSuchKey. */
+export function isS3NotFound(error: any): boolean {
+  return (
+    error.name === "NotFound" ||
+    error.name === "NoSuchKey" ||
+    error.$metadata?.httpStatusCode === 404
+  );
+}
+
+export async function s3ObjectExists(
+  client: S3Client,
+  bucketName: string,
+  key: string,
+): Promise<boolean> {
+  try {
+    await client.send(new HeadObjectCommand({ Bucket: bucketName, Key: key }));
+    return true;
+  } catch (error: any) {
+    if (isS3NotFound(error)) {
+      return false;
+    }
+    throw error;
+  }
+}
+
+/** True when the version was uploaded with the skus/<sku>/ folder layout. */
+export async function versionHasSkuSupport(
+  client: S3Client,
+  bucketName: string,
+  prefix: string,
+  version: string,
+): Promise<boolean> {
+  const response = await client.send(
+    new ListObjectsV2Command({
+      Bucket: bucketName,
+      Prefix: `${prefix}/${version}/skus/`,
+      MaxKeys: 1,
+    }),
+  );
+  return (response.Contents?.length ?? 0) > 0;
+}
diff --git a/test/sync-releases.test.ts b/test/sync-releases.test.ts
index c0f1a92..51b3fbd 100644
--- a/test/sync-releases.test.ts
+++ b/test/sync-releases.test.ts
@@ -4,12 +4,20 @@ import {
   ListObjectsV2Command,
   S3Client,
 } from "@aws-sdk/client-s3";
-import { describe, expect, beforeEach, it } from "vitest";
+import { PrismaClient } from "@prisma/client";
+import { afterEach, describe, expect, beforeEach, it, vi } from "vitest";
 
-import { collectReleaseArtifacts, syncReleases } from "../scripts/sync-releases";
+import {
+  collectReleaseArtifacts,
+  createAtDefaultRollout,
+  scheduleReleaseSync,
+  syncReleases,
+  UPLOAD_SETTLE_MS,
+  type ReleaseDecider,
+  type ReleaseType,
+} from "../src/release-sync";
 import { otaFileForPrefix } from "../src/skus";
 
-type ReleaseType = string;
 import { createAsyncIterable, s3Mock, testPrisma } from "./setup";
 
 const DEFAULT_SKU = "jetkvm-v2";
@@ -20,9 +28,27 @@ const SYNC_BUCKET = "test-bucket";
 const SYNC_BASE_URL = "https://cdn.test.com";
 const syncS3Client = new S3Client({});
 
-function mockS3ListVersions(prefix: ReleaseType, versions: string[]) {
-  s3Mock.on(ListObjectsV2Command, { Prefix: `${prefix}/` }).resolves({
-    CommonPrefixes: versions.map(v => ({ Prefix: `${prefix}/${v}/` })),
+/** Makes the settle check see one object under the version uploaded at `at`. */
+function mockS3UploadedAt(prefix: ReleaseType, version: string, at: Date) {
+  s3Mock.on(ListObjectsV2Command, { Prefix: `${prefix}/${version}/` }).resolves({
+    Contents: [{ Key: `${prefix}/${version}/${otaFileForPrefix(prefix)}`, LastModified: at }],
+  });
+}
+
+/** Lists `versions` under `prefix`, one page per inner array, as R2 does past 1,000 keys. */
+function mockS3ListVersions(prefix: ReleaseType, ...pages: string[][]) {
+  pages.forEach((versions, index) => {
+    const last = index === pages.length - 1;
+    s3Mock
+      .on(ListObjectsV2Command, {
+        Prefix: `${prefix}/`,
+        ContinuationToken: index === 0 ? undefined : `page-${index}`,
+      })
+      .resolves({
+        CommonPrefixes: versions.map(v => ({ Prefix: `${prefix}/${v}/` })),
+        IsTruncated: !last,
+        NextContinuationToken: last ? undefined : `page-${index + 1}`,
+      });
   });
 }
 
@@ -56,14 +82,17 @@ function mockS3SkuVersion(
   });
 }
 
-describe("sync-releases script", () => {
-  beforeEach(() => {
-    s3Mock.reset();
-    s3Mock
-      .on(HeadObjectCommand)
-      .rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
-  });
+beforeEach(() => {
+  s3Mock.reset();
+  s3Mock
+    .on(HeadObjectCommand)
+    .rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
+  // Listings a test does not stub (other prefixes, the upload settle check) see
+  // an empty folder. More specific .on(..., { Prefix }) stubs registered later win.
+  s3Mock.on(ListObjectsV2Command).resolves({ Contents: [] });
+});
 
+describe("syncReleases", () => {
   it("marks legacy app artifacts compatible with the default SKU only", async () => {
     mockS3HashFile("app", "9.9.1", "legacy-app-hash");
 
@@ -178,16 +207,25 @@ describe("sync-releases script", () => {
 
     mockS3ListVersions("app", [version, "10.0.0-beta.1"]);
     mockS3ListVersions("system", [version]);
-    mockS3ListVersions("mini", []);
     mockS3HashFile("app", version, "app-hash");
     mockS3SkuVersion("system", version, DEFAULT_SKU, "system-hash-v2");
     mockS3SkuVersion("system", version, SDMMC_SKU, "system-hash-sdmmc");
 
-    await syncReleases(
+    const stats = await syncReleases(
       { prisma: testPrisma, s3Client: syncS3Client },
       { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      createAtDefaultRollout,
     );
 
+    expect(stats).toEqual({
+      created: 1,
+      "already-synced": 1,
+      uploading: 0,
+      "no-artifacts": 0,
+      skipped: 0,
+      aborted: 0,
+    });
+
     const appRelease = await testPrisma.release.findUniqueOrThrow({
       where: { version_type: { version, type: "app" } },
       include: { artifacts: true },
@@ -220,4 +258,225 @@ describe("sync-releases script", () => {
     // Prereleases are filtered out by listStableVersions.
     expect(prerelease).toBeNull();
   });
+
+  it("registers versions from every page of a truncated listing", async () => {
+    const firstPage = "9.9.20";
+    const secondPage = "9.9.21";
+    mockS3ListVersions("app", [firstPage], [secondPage]);
+    mockS3HashFile("app", firstPage, "first-page-hash");
+    mockS3HashFile("app", secondPage, "second-page-hash");
+
+    const stats = await syncReleases(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      createAtDefaultRollout,
+    );
+
+    expect(stats).toMatchObject({ created: 2 });
+    const created = await testPrisma.release.findMany({
+      where: { type: "app", version: { in: [firstPage, secondPage] } },
+      orderBy: { version: "asc" },
+    });
+    expect(created.map(release => release.version)).toEqual([firstPage, secondPage]);
+  });
+
+  it("defers a version whose objects changed within the settle window", async () => {
+    const fresh = "9.9.10";
+    const settled = "9.9.11";
+    mockS3ListVersions("app", [fresh, settled]);
+    mockS3HashFile("app", fresh, "fresh-hash");
+    mockS3HashFile("app", settled, "settled-hash");
+    mockS3UploadedAt("app", fresh, new Date(Date.now() - 60 * 1000));
+    mockS3UploadedAt("app", settled, new Date(Date.now() - UPLOAD_SETTLE_MS - 60 * 1000));
+
+    const stats = await syncReleases(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL, uploadSettleMs: UPLOAD_SETTLE_MS },
+      createAtDefaultRollout,
+    );
+
+    expect(stats).toMatchObject({ created: 1, uploading: 1 });
+    expect(
+      await testPrisma.release.findUnique({
+        where: { version_type: { version: fresh, type: "app" } },
+      }),
+    ).toBeNull();
+    // The settle listing must come after the artifact scan, so an upload that
+    // overlaps the scan is seen. The deferred version was therefore scanned.
+    const calls = s3Mock.calls().map(call => call.args[0].input as { Key?: string; Prefix?: string });
+    const scanIndex = calls.findIndex(input => input.Key === `app/${fresh}/jetkvm_app.sha256`);
+    const settleIndex = calls.findIndex(input => input.Prefix === `app/${fresh}/`);
+    expect(scanIndex).toBeGreaterThanOrEqual(0);
+    expect(settleIndex).toBeGreaterThan(scanIndex);
+    expect(
+      await testPrisma.release.findUnique({
+        where: { version_type: { version: settled, type: "app" } },
+      }),
+    ).toMatchObject({ hash: "settled-hash" });
+  });
+
+  it("honours the decider's rollout, skip and abort answers", async () => {
+    mockS3ListVersions("app", ["9.9.5", "9.9.6", "9.9.7"]);
+    mockS3ListVersions("system", ["9.9.5"]);
+    for (const version of ["9.9.5", "9.9.6", "9.9.7"]) {
+      mockS3HashFile("app", version, `app-hash-${version}`);
+    }
+    mockS3HashFile("system", "9.9.5", "system-hash");
+
+    const answers: Record<string, Awaited<ReturnType<ReleaseDecider>>> = {
+      "app 9.9.5": { kind: "create", rolloutPercentage: 42 },
+      "app 9.9.6": { kind: "skip" },
+      "app 9.9.7": { kind: "abort" },
+    };
+    const decide: ReleaseDecider = async (type, version) => answers[`${type} ${version}`];
+
+    const stats = await syncReleases(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      decide,
+    );
+
+    expect(stats).toMatchObject({ created: 1, skipped: 1, aborted: 1 });
+
+    const created = await testPrisma.release.findUniqueOrThrow({
+      where: { version_type: { version: "9.9.5", type: "app" } },
+    });
+    expect(created.rolloutPercentage).toBe(42);
+
+    const notCreated = await testPrisma.release.findMany({
+      where: {
+        OR: [
+          { version: "9.9.6", type: "app" },
+          { version: "9.9.7", type: "app" },
+          // Abort stops the whole run, so system is never reached.
+          { version: "9.9.5", type: "system" },
+        ],
+      },
+    });
+    expect(notCreated).toEqual([]);
+  });
+
+  it("treats a release created by another instance mid-run as already synced", async () => {
+    const version = "9.9.8";
+    mockS3ListVersions("app", [version]);
+    mockS3HashFile("app", version, "app-hash");
+
+    // Simulate the race: the known-versions query sees nothing, but by the
+    // time this instance inserts, the row is there (written here up front so
+    // the real unique constraint fires on create).
+    await testPrisma.release.create({
+      data: {
+        version,
+        type: "app",
+        rolloutPercentage: 10,
+        url: "https://cdn.test.com/other-instance",
+        hash: "other-instance-hash",
+      },
+    });
+    const racingPrisma = {
+      release: {
+        findMany: async () => [],
+        create: (args: unknown) => testPrisma.release.create(args as any),
+      },
+    } as unknown as PrismaClient;
+
+    const stats = await syncReleases(
+      { prisma: racingPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      createAtDefaultRollout,
+    );
+
+    expect(stats).toMatchObject({ created: 0, "already-synced": 1 });
+    const release = await testPrisma.release.findUniqueOrThrow({
+      where: { version_type: { version, type: "app" } },
+    });
+    expect(release.url).toBe("https://cdn.test.com/other-instance");
+  });
+});
+
+describe("scheduleReleaseSync", () => {
+  const INTERVAL_MS = 1000;
+  let stop: (() => void) | undefined;
+
+  beforeEach(() => {
+    // Only the scheduler's own timer is faked; DB and S3 mock I/O stay real.
+    vi.useFakeTimers({ toFake: ["setInterval", "clearInterval"] });
+  });
+
+  afterEach(() => {
+    stop?.();
+    stop = undefined;
+    vi.useRealTimers();
+    vi.restoreAllMocks();
+  });
+
+  it("runs at start, keeps the schedule after a failed run and creates on the next tick", async () => {
+    const version = "9.9.9";
+    s3Mock
+      .on(ListObjectsV2Command, { Prefix: "app/" })
+      .rejectsOnce(new Error("R2 unavailable"))
+      .resolves({ CommonPrefixes: [{ Prefix: `app/${version}/` }] });
+    mockS3HashFile("app", version, "app-hash");
+    const errorLog = vi.spyOn(console, "error").mockImplementation(() => {});
+
+    stop = scheduleReleaseSync(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      INTERVAL_MS,
+    );
+
+    await vi.waitFor(() => expect(errorLog).toHaveBeenCalledOnce());
+    expect(
+      await testPrisma.release.findUnique({
+        where: { version_type: { version, type: "app" } },
+      }),
+    ).toBeNull();
+
+    await vi.advanceTimersByTimeAsync(INTERVAL_MS);
+
+    await vi.waitFor(async () => {
+      const release = await testPrisma.release.findUnique({
+        where: { version_type: { version, type: "app" } },
+      });
+      expect(release?.rolloutPercentage).toBe(10);
+    });
+  });
+
+  it("skips a tick while the previous run is still in progress", async () => {
+    let finishFirstRun!: () => void;
+    const firstListing = new Promise<{ CommonPrefixes: never[] }>(resolve => {
+      finishFirstRun = () => resolve({ CommonPrefixes: [] });
+    });
+    s3Mock
+      .on(ListObjectsV2Command, { Prefix: "app/" })
+      .callsFakeOnce(() => firstListing)
+      .resolves({ CommonPrefixes: [] });
+    const warnLog = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+    stop = scheduleReleaseSync(
+      { prisma: testPrisma, s3Client: syncS3Client },
+      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+      INTERVAL_MS,
+    );
+
+    // Two ticks fire while the first run is still waiting on R2.
+    await vi.advanceTimersByTimeAsync(INTERVAL_MS * 2);
+
+    expect(warnLog).toHaveBeenCalledTimes(2);
+    // The stalled run's list call is the only R2 traffic so far: the skipped
+    // ticks did not start a second walk of the bucket.
+    expect(s3Mock.commandCalls(ListObjectsV2Command)).toHaveLength(1);
+
+    finishFirstRun();
+    await vi.waitFor(() =>
+      expect(s3Mock.commandCalls(ListObjectsV2Command)).toHaveLength(3),
+    );
+
+    // The next tick after the run completed starts a fresh run.
+    await vi.advanceTimersByTimeAsync(INTERVAL_MS);
+    await vi.waitFor(() =>
+      expect(s3Mock.commandCalls(ListObjectsV2Command)).toHaveLength(6),
+    );
+    expect(warnLog).toHaveBeenCalledTimes(2);
+  });
 });

From 68c4dfcc8f47256c0b3f9a8d955b2841469da9cc Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Sat, 19 Sep 2026 14:30:39 +0200
Subject: [PATCH 03/12] Release sync: a unique violation is only a race when
 the row exists (#82)

* fix(release-sync): only treat a unique violation as a race when the row exists

Sync caught every P2002 from the release insert as "created concurrently
elsewhere". On staging the id sequences were behind the rows after a data
import, so each insert failed on the primary key, was logged as a race, and
left nothing in the table.

After a unique violation, createRelease now looks the (version, type) row
up. Present means another instance registered it first; absent means the
insert really failed, and the error is rethrown with the type and version
in its message so the scheduled run log names the release.

* fix(release-sync): name the release in every per-version failure

Wrapping only the insert error left the row lookup, and the S3 scan
before it, free to escape without the type and version. syncReleases now
wraps whatever createRelease throws for a version.
---
 src/release-sync.ts        | 24 +++++++++++++++++++++---
 test/sync-releases.test.ts | 37 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/src/release-sync.ts b/src/release-sync.ts
index 2911427..7a2f647 100644
--- a/src/release-sync.ts
+++ b/src/release-sync.ts
@@ -216,6 +216,18 @@ async function listSyncedVersions(prisma: PrismaClient, type: ReleaseType): Prom
   return new Set(releases.map(release => release.version));
 }
 
+async function releaseExists(
+  prisma: PrismaClient,
+  type: ReleaseType,
+  version: string,
+): Promise<boolean> {
+  const release = await prisma.release.findUnique({
+    where: { version_type: { version, type } },
+    select: { id: true },
+  });
+  return release !== null;
+}
+
 function isUniqueViolation(error: unknown): boolean {
   return (
     error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002"
@@ -279,8 +291,10 @@ async function createRelease(
     });
   } catch (error) {
     // Another API instance can win the race between the version listing and
-    // this insert. The row it wrote is the one we wanted, so treat it as synced.
-    if (isUniqueViolation(error)) {
+    // this insert, in which case the row it wrote is the one we wanted. Any
+    // other unique violation (a stale id sequence after a data import, say)
+    // leaves no row and is a real failure.
+    if (isUniqueViolation(error) && (await releaseExists(clients.prisma, type, version))) {
       console.log(`[sync-releases] ${type} ${version}: created concurrently elsewhere, skipping`);
       return "already-synced";
     }
@@ -326,9 +340,13 @@ export async function syncReleases(
     ]);
 
     for (const version of versions) {
+      // Name the release in any failure, whichever step raised it, so the
+      // scheduled run log does not need to be traced back to a version.
       const outcome = synced.has(version)
         ? "already-synced"
-        : await createRelease(clients, config, decide, type, version);
+        : await createRelease(clients, config, decide, type, version).catch((error: unknown) => {
+            throw new Error(`[sync-releases] ${type} ${version}: sync failed`, { cause: error });
+          });
       stats[outcome]++;
 
       if (outcome === "aborted") {
diff --git a/test/sync-releases.test.ts b/test/sync-releases.test.ts
index 51b3fbd..aa29e18 100644
--- a/test/sync-releases.test.ts
+++ b/test/sync-releases.test.ts
@@ -4,7 +4,7 @@ import {
   ListObjectsV2Command,
   S3Client,
 } from "@aws-sdk/client-s3";
-import { PrismaClient } from "@prisma/client";
+import { Prisma, PrismaClient } from "@prisma/client";
 import { afterEach, describe, expect, beforeEach, it, vi } from "vitest";
 
 import {
@@ -376,6 +376,7 @@ describe("syncReleases", () => {
     const racingPrisma = {
       release: {
         findMany: async () => [],
+        findUnique: (args: unknown) => testPrisma.release.findUnique(args as any),
         create: (args: unknown) => testPrisma.release.create(args as any),
       },
     } as unknown as PrismaClient;
@@ -392,6 +393,40 @@ describe("syncReleases", () => {
     });
     expect(release.url).toBe("https://cdn.test.com/other-instance");
   });
+
+  it("fails on a unique violation that left no release row behind", async () => {
+    const version = "9.9.9";
+    mockS3ListVersions("app", [version]);
+    mockS3HashFile("app", version, "app-hash");
+
+    // A stale id sequence (a database restored from a dump) makes the insert
+    // fail on the primary key with the same error code as the race above,
+    // but no row for this version exists afterwards.
+    const stalePrisma = {
+      release: {
+        findMany: async () => [],
+        findUnique: async () => null,
+        create: async () => {
+          throw new Prisma.PrismaClientKnownRequestError("Unique constraint failed", {
+            code: "P2002",
+            clientVersion: "test",
+            meta: { target: ["id"] },
+          });
+        },
+      },
+    } as unknown as PrismaClient;
+
+    await expect(
+      syncReleases(
+        { prisma: stalePrisma, s3Client: syncS3Client },
+        { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+        createAtDefaultRollout,
+      ),
+    ).rejects.toMatchObject({
+      message: "[sync-releases] app 9.9.9: sync failed",
+      cause: expect.objectContaining({ code: "P2002" }),
+    });
+  });
 });
 
 describe("scheduleReleaseSync", () => {

From 02ee5a82cc8d141d791aa999c58565ddb0248750 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Sat, 19 Sep 2026 15:08:07 +0200
Subject: [PATCH 04/12] feat(releases): POST /releases/sync for the upload
 script (#83)

The scheduled tick found a new version at most 30 minutes plus the settle
window after upload. The upload script knows when its last object is
written, so it can trigger the sync itself.

One runner now owns the in-progress flag and serves both the timer and
the endpoint. The endpoint is guarded by RELEASE_SYNC_TOKEN as a bearer
token, compared in constant time, and is not registered without it. It
runs with the settle window off and answers with the per-outcome counts,
or 409 while a run is in progress.
---
 .env.example               |  4 +++
 src/auth.ts                | 15 +++++++++++
 src/errors.ts              |  8 ++++++
 src/index.ts               | 18 ++++++++++---
 src/release-sync.ts        | 54 ++++++++++++++++++++++++++------------
 src/releases.ts            | 19 +++++++++++++-
 test/auth.test.ts          | 31 ++++++++++++++++++++++
 test/sync-releases.test.ts | 53 ++++++++++++++++++++++++++++++-------
 8 files changed, 171 insertions(+), 31 deletions(-)
 create mode 100644 test/auth.test.ts

diff --git a/.env.example b/.env.example
index a7a246a..0ad2520 100644
--- a/.env.example
+++ b/.env.example
@@ -40,3 +40,7 @@ REAL_IP_HEADER=
 
 # ICE Servers for WebRTC, split by comma (e.g. stun:stun.l.google.com:19302,stun:stun1.l.google.com:19302)
 ICE_SERVERS=
+
+# Bearer token for POST /releases/sync (the upload script calls it after its last write).
+# Leave empty to disable the endpoint.
+RELEASE_SYNC_TOKEN=
diff --git a/src/auth.ts b/src/auth.ts
index 8985196..5856cca 100644
--- a/src/auth.ts
+++ b/src/auth.ts
@@ -1,3 +1,4 @@
+import { createHash, timingSafeEqual } from "crypto";
 import { type NextFunction, type Request, type Response } from "express";
 import * as jose from "jose";
 import { UnauthorizedError } from "./errors";
@@ -56,3 +57,17 @@ export const authenticated = async (req: Request, res: Response, next: NextFunct
 
   next();
 };
+
+const sha256 = (value: string) => createHash("sha256").update(value).digest();
+
+/** Guards a route with one static bearer token, compared in constant time. */
+export const bearerToken = (expected: string) => {
+  const expectedDigest = sha256(expected);
+  return (req: Request, res: Response, next: NextFunction) => {
+    const presented = req.headers.authorization?.match(/^Bearer (.+)$/)?.[1];
+    if (!presented || !timingSafeEqual(sha256(presented), expectedDigest)) {
+      throw new UnauthorizedError("Invalid bearer token");
+    }
+    next();
+  };
+};
diff --git a/src/errors.ts b/src/errors.ts
index 07f868b..936747d 100644
--- a/src/errors.ts
+++ b/src/errors.ts
@@ -31,6 +31,14 @@ export class ForbiddenError extends HttpError {
   }
 }
 
+export class ConflictError extends HttpError {
+  constructor(message?: string, code?: string) {
+    super(409, message);
+    this.name = "ConflictError";
+    this.code = code;
+  }
+}
+
 export class NotFoundError extends HttpError {
   constructor(message?: string, code?: string) {
     super(404, message);
diff --git a/src/index.ts b/src/index.ts
index 2a235c1..5d677e9 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -11,10 +11,10 @@ import * as Webrtc from "./webrtc";
 import * as Releases from "./releases";
 
 import { HttpError } from "./errors";
-import { authenticated } from "./auth";
+import { authenticated, bearerToken } from "./auth";
 import { prisma } from "./db";
 import { baseUrl, bucketName, s3Client } from "./s3";
-import { scheduleReleaseSync } from "./release-sync";
+import { createReleaseSyncRunner, scheduleReleaseSync } from "./release-sync";
 import { initializeWebRTCSignaling } from "./webrtc-signaling";
 
 declare global {
@@ -49,6 +49,7 @@ declare global {
       ICE_SERVERS: string;
 
       ALLOWED_IDENTITIES?: string;
+      RELEASE_SYNC_TOKEN?: string;
     }
   }
 }
@@ -116,12 +117,23 @@ app.get(
   },
 );
 
+// One sync at a time, shared by the scheduled tick below and the upload
+// script's POST /releases/sync. The route exists only with a token configured.
+const releaseSync = createReleaseSyncRunner({ prisma, s3Client }, { bucketName, baseUrl });
+
 app.get("/releases", Releases.Retrieve);
 app.get(
   "/releases/system_recovery/latest",
   Releases.RetrieveLatestSystemRecovery,
 );
 app.get("/releases/app/latest", Releases.RetrieveLatestApp);
+if (process.env.RELEASE_SYNC_TOKEN) {
+  app.post(
+    "/releases/sync",
+    bearerToken(process.env.RELEASE_SYNC_TOKEN),
+    Releases.Sync(releaseSync),
+  );
+}
 
 app.get("/devices", authenticated, Devices.List);
 app.get("/devices/:id", authenticated, Devices.Retrieve);
@@ -222,4 +234,4 @@ const server = app.listen(PORT, () => {
 initializeWebRTCSignaling(server);
 
 // Register new R2 releases at the default rollout, now and every 30 minutes.
-scheduleReleaseSync({ prisma, s3Client }, { bucketName, baseUrl });
+scheduleReleaseSync(releaseSync);
diff --git a/src/release-sync.ts b/src/release-sync.ts
index 7a2f647..960c6c8 100644
--- a/src/release-sync.ts
+++ b/src/release-sync.ts
@@ -44,6 +44,8 @@ export type ReleaseOutcome =
   | "skipped"
   | "aborted";
 
+export type SyncStats = Record<ReleaseOutcome, number>;
+
 /** Settle window for unattended runs; shorter than the interval, so it costs at most one tick. */
 export const UPLOAD_SETTLE_MS = 10 * 60 * 1000;
 
@@ -322,8 +324,8 @@ export async function syncReleases(
   clients: SyncClients,
   config: SyncConfig,
   decide: ReleaseDecider,
-): Promise<Record<ReleaseOutcome, number>> {
-  const stats: Record<ReleaseOutcome, number> = {
+): Promise<SyncStats> {
+  const stats: SyncStats = {
     created: 0,
     "already-synced": 0,
     uploading: 0,
@@ -369,38 +371,56 @@ export async function syncReleases(
 
 const RELEASE_SYNC_INTERVAL_MS = 30 * 60 * 1000;
 
+/** Runs one unattended sync, or resolves to "busy" while another run is in progress. */
+export type ReleaseSyncRunner = (
+  options?: Pick<SyncConfig, "uploadSettleMs">,
+) => Promise<SyncStats | "busy">;
+
 /**
- * Runs the sync now and then every `intervalMs`, inside the API process.
- * A tick that fires while the previous run is still going is skipped, not
- * queued. A failed run is logged and the schedule continues, so one bad R2
- * or DB response never stops future syncs.
- * Returns a function that stops the schedule.
+ * One in-process sync at a time, shared by every trigger (the timer and the
+ * HTTP endpoint): a second run would only walk the bucket again and lose the
+ * (version, type) race to the first.
  */
-export function scheduleReleaseSync(
+export function createReleaseSyncRunner(
   clients: SyncClients,
   config: SyncConfig,
-  intervalMs: number = RELEASE_SYNC_INTERVAL_MS,
-): () => void {
+): ReleaseSyncRunner {
   let running = false;
 
-  const run = async () => {
+  return async (options = {}) => {
     if (running) {
-      console.warn("[sync-releases] previous run still in progress, skipping this tick");
-      return;
+      return "busy";
     }
     running = true;
     try {
-      await syncReleases(
+      return await syncReleases(
         clients,
-        { uploadSettleMs: UPLOAD_SETTLE_MS, ...config },
+        { uploadSettleMs: UPLOAD_SETTLE_MS, ...config, ...options },
         createAtDefaultRollout,
       );
-    } catch (error) {
-      console.error("[sync-releases] scheduled run failed", error);
     } finally {
       running = false;
     }
   };
+}
+
+/**
+ * Runs the sync now and then every `intervalMs`. A failed run is logged and
+ * the schedule continues. Returns a function that stops the schedule.
+ */
+export function scheduleReleaseSync(
+  runner: ReleaseSyncRunner,
+  intervalMs: number = RELEASE_SYNC_INTERVAL_MS,
+): () => void {
+  const run = async () => {
+    try {
+      if ((await runner()) === "busy") {
+        console.warn("[sync-releases] previous run still in progress, skipping this tick");
+      }
+    } catch (error) {
+      console.error("[sync-releases] scheduled run failed", error);
+    }
+  };
 
   void run();
   const timer = setInterval(run, intervalMs);
diff --git a/src/releases.ts b/src/releases.ts
index 33d5323..1763513 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -1,6 +1,7 @@
 import { Request, Response } from "express";
 import { prisma } from "./db";
-import { BadRequestError, InternalServerError, NotFoundError } from "./errors";
+import { BadRequestError, ConflictError, InternalServerError, NotFoundError } from "./errors";
+import type { ReleaseSyncRunner } from "./release-sync";
 import semver from "semver";
 
 import { GetObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3";
@@ -710,3 +711,19 @@ function latestArtifactRedirect(kind: OtaKind) {
 }
 
 export const RetrieveLatestApp = latestArtifactRedirect("app");
+
+/**
+ * POST /releases/sync: register every stable R2 version missing from the DB
+ * and answer with the per-outcome counts. Made for the upload script's last
+ * step, so the settle window is off: the caller vouches its last object is
+ * written.
+ */
+export function Sync(runner: ReleaseSyncRunner) {
+  return async (req: Request, res: Response) => {
+    const stats = await runner({ uploadSettleMs: 0 });
+    if (stats === "busy") {
+      throw new ConflictError("A release sync is already in progress");
+    }
+    return res.json(stats);
+  };
+}
diff --git a/test/auth.test.ts b/test/auth.test.ts
new file mode 100644
index 0000000..2adcc51
--- /dev/null
+++ b/test/auth.test.ts
@@ -0,0 +1,31 @@
+import type { Request, Response } from "express";
+import { describe, expect, it, vi } from "vitest";
+
+import { bearerToken } from "../src/auth";
+import { UnauthorizedError } from "../src/errors";
+
+describe("bearerToken", () => {
+  const guard = bearerToken("release-sync-secret");
+  const res = {} as Response;
+
+  function request(authorization?: string): Request {
+    return { headers: { authorization } } as unknown as Request;
+  }
+
+  it("calls next for the configured token", () => {
+    const next = vi.fn();
+    guard(request("Bearer release-sync-secret"), res, next);
+    expect(next).toHaveBeenCalledOnce();
+  });
+
+  it.each([
+    ["no header", undefined],
+    ["wrong token", "Bearer nope"],
+    ["missing scheme", "release-sync-secret"],
+    ["prefix of the token", "Bearer release-sync"],
+  ])("rejects %s without calling next", (_label, authorization) => {
+    const next = vi.fn();
+    expect(() => guard(request(authorization), res, next)).toThrow(UnauthorizedError);
+    expect(next).not.toHaveBeenCalled();
+  });
+});
diff --git a/test/sync-releases.test.ts b/test/sync-releases.test.ts
index aa29e18..fd55f52 100644
--- a/test/sync-releases.test.ts
+++ b/test/sync-releases.test.ts
@@ -5,17 +5,21 @@ import {
   S3Client,
 } from "@aws-sdk/client-s3";
 import { Prisma, PrismaClient } from "@prisma/client";
+import type { Request, Response } from "express";
 import { afterEach, describe, expect, beforeEach, it, vi } from "vitest";
 
 import {
   collectReleaseArtifacts,
   createAtDefaultRollout,
+  createReleaseSyncRunner,
   scheduleReleaseSync,
   syncReleases,
   UPLOAD_SETTLE_MS,
   type ReleaseDecider,
   type ReleaseType,
 } from "../src/release-sync";
+import { Sync } from "../src/releases";
+import { ConflictError } from "../src/errors";
 import { otaFileForPrefix } from "../src/skus";
 
 import { createAsyncIterable, s3Mock, testPrisma } from "./setup";
@@ -92,6 +96,13 @@ beforeEach(() => {
   s3Mock.on(ListObjectsV2Command).resolves({ Contents: [] });
 });
 
+function newRunner() {
+  return createReleaseSyncRunner(
+    { prisma: testPrisma, s3Client: syncS3Client },
+    { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
+  );
+}
+
 describe("syncReleases", () => {
   it("marks legacy app artifacts compatible with the default SKU only", async () => {
     mockS3HashFile("app", "9.9.1", "legacy-app-hash");
@@ -454,11 +465,7 @@ describe("scheduleReleaseSync", () => {
     mockS3HashFile("app", version, "app-hash");
     const errorLog = vi.spyOn(console, "error").mockImplementation(() => {});
 
-    stop = scheduleReleaseSync(
-      { prisma: testPrisma, s3Client: syncS3Client },
-      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
-      INTERVAL_MS,
-    );
+    stop = scheduleReleaseSync(newRunner(), INTERVAL_MS);
 
     await vi.waitFor(() => expect(errorLog).toHaveBeenCalledOnce());
     expect(
@@ -488,11 +495,7 @@ describe("scheduleReleaseSync", () => {
       .resolves({ CommonPrefixes: [] });
     const warnLog = vi.spyOn(console, "warn").mockImplementation(() => {});
 
-    stop = scheduleReleaseSync(
-      { prisma: testPrisma, s3Client: syncS3Client },
-      { bucketName: SYNC_BUCKET, baseUrl: SYNC_BASE_URL },
-      INTERVAL_MS,
-    );
+    stop = scheduleReleaseSync(newRunner(), INTERVAL_MS);
 
     // Two ticks fire while the first run is still waiting on R2.
     await vi.advanceTimersByTimeAsync(INTERVAL_MS * 2);
@@ -515,3 +518,33 @@ describe("scheduleReleaseSync", () => {
     expect(warnLog).toHaveBeenCalledTimes(2);
   });
 });
+
+describe("Sync handler", () => {
+  const request = {} as Request;
+
+  it("registers a version uploaded moments ago and answers with the counts", async () => {
+    const version = "9.9.12";
+    mockS3ListVersions("app", [version]);
+    mockS3HashFile("app", version, "fresh-hash");
+    // Inside the settle window: the scheduled tick would defer this version,
+    // the endpoint trusts the caller and registers it.
+    mockS3UploadedAt("app", version, new Date(Date.now() - 60 * 1000));
+    const res = { json: vi.fn() } as unknown as Response;
+
+    await Sync(newRunner())(request, res);
+
+    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ created: 1, uploading: 0 }));
+    const release = await testPrisma.release.findUniqueOrThrow({
+      where: { version_type: { version, type: "app" } },
+    });
+    expect(release.rolloutPercentage).toBe(10);
+  });
+
+  it("answers 409 while a run is already in progress", async () => {
+    const busyRunner = vi.fn().mockResolvedValue("busy");
+
+    await expect(Sync(busyRunner)(request, { json: vi.fn() } as unknown as Response)).rejects.toThrow(
+      ConflictError,
+    );
+  });
+});

From 2abf19dd0b3c2aa469f9716e89d457bda4e79b95 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Sat, 19 Sep 2026 15:19:55 +0200
Subject: [PATCH 05/12] fix(auth): accept the bearer scheme in any case (#85)

Scheme names are case-insensitive (RFC 9110). The token is still
compared exactly.
---
 src/auth.ts       |  3 ++-
 test/auth.test.ts | 14 +++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/auth.ts b/src/auth.ts
index 5856cca..96c895d 100644
--- a/src/auth.ts
+++ b/src/auth.ts
@@ -64,7 +64,8 @@ const sha256 = (value: string) => createHash("sha256").update(value).digest();
 export const bearerToken = (expected: string) => {
   const expectedDigest = sha256(expected);
   return (req: Request, res: Response, next: NextFunction) => {
-    const presented = req.headers.authorization?.match(/^Bearer (.+)$/)?.[1];
+    // The scheme name is case-insensitive (RFC 9110); the token is not.
+    const presented = req.headers.authorization?.match(/^Bearer +(.+)$/i)?.[1];
     if (!presented || !timingSafeEqual(sha256(presented), expectedDigest)) {
       throw new UnauthorizedError("Invalid bearer token");
     }
diff --git a/test/auth.test.ts b/test/auth.test.ts
index 2adcc51..d324c3c 100644
--- a/test/auth.test.ts
+++ b/test/auth.test.ts
@@ -12,17 +12,21 @@ describe("bearerToken", () => {
     return { headers: { authorization } } as unknown as Request;
   }
 
-  it("calls next for the configured token", () => {
-    const next = vi.fn();
-    guard(request("Bearer release-sync-secret"), res, next);
-    expect(next).toHaveBeenCalledOnce();
-  });
+  it.each(["Bearer release-sync-secret", "bearer release-sync-secret", "BEARER  release-sync-secret"])(
+    "calls next for %j",
+    authorization => {
+      const next = vi.fn();
+      guard(request(authorization), res, next);
+      expect(next).toHaveBeenCalledOnce();
+    },
+  );
 
   it.each([
     ["no header", undefined],
     ["wrong token", "Bearer nope"],
     ["missing scheme", "release-sync-secret"],
     ["prefix of the token", "Bearer release-sync"],
+    ["token in a different case", "Bearer RELEASE-SYNC-SECRET"],
   ])("rejects %s without calling next", (_label, authorization) => {
     const next = vi.fn();
     expect(() => guard(request(authorization), res, next)).toThrow(UnauthorizedError);

From 6d5da4fa7a28d4c0fbd0134379761d1f5813d325 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Sat, 19 Sep 2026 15:21:02 +0200
Subject: [PATCH 06/12] fix(releases): scope the sync endpoint's settle bypass
 to one version (#84)

The endpoint turned the settle window off for every version in the
bucket. A call made while a different upload was still running
registered that upload half done, and sync never revisits a row.

The body now names the version the caller finished, and only that
version skips the window. Without a body the call is a normal tick.
---
 src/release-sync.ts        |  7 +++-
 src/releases.ts            | 30 ++++++++++++---
 test/sync-releases.test.ts | 75 ++++++++++++++++++++++++++++----------
 3 files changed, 86 insertions(+), 26 deletions(-)

diff --git a/src/release-sync.ts b/src/release-sync.ts
index 960c6c8..b176040 100644
--- a/src/release-sync.ts
+++ b/src/release-sync.ts
@@ -26,6 +26,8 @@ export interface SyncConfig {
    * (an operator at a terminal can judge the artifact list for themselves).
    */
   uploadSettleMs?: number;
+  /** One version whose upload the caller vouches is complete: it skips the settle check. */
+  settled?: { type: ReleaseType; version: string };
 }
 
 export interface ReleaseArtifactInput {
@@ -248,7 +250,8 @@ async function createRelease(
   // Listed after the artifact scan on purpose: an upload that was active at
   // any point during the scan leaves an object newer than the window, so the
   // snapshot above is discarded rather than registered.
-  if (config.uploadSettleMs) {
+  const vouched = config.settled?.type === type && config.settled.version === version;
+  if (config.uploadSettleMs && !vouched) {
     const newest = await newestUploadTime(clients.s3Client, config.bucketName, type, version);
     if (newest && Date.now() - newest.getTime() < config.uploadSettleMs) {
       console.log(
@@ -373,7 +376,7 @@ const RELEASE_SYNC_INTERVAL_MS = 30 * 60 * 1000;
 
 /** Runs one unattended sync, or resolves to "busy" while another run is in progress. */
 export type ReleaseSyncRunner = (
-  options?: Pick<SyncConfig, "uploadSettleMs">,
+  options?: Pick<SyncConfig, "settled">,
 ) => Promise<SyncStats | "busy">;
 
 /**
diff --git a/src/releases.ts b/src/releases.ts
index 1763513..30310fd 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -21,6 +21,7 @@ import {
   isKnownSku,
   legacyCompatibleSkus,
   otaArtifacts,
+  OTA_PREFIXES,
   type OtaKind,
   type Artifact,
 } from "./skus";
@@ -72,8 +73,12 @@ type RetrieveQuery = z.infer<typeof retrieveQuerySchema>;
  * Parses query parameters and converts ZodError to BadRequestError.
  */
 function parseQuery<T>(schema: z.ZodSchema<T>, req: Request): T {
+  return parseOrBadRequest(schema, req.query);
+}
+
+function parseOrBadRequest<T>(schema: z.ZodSchema<T>, input: unknown): T {
   try {
-    return schema.parse(req.query);
+    return schema.parse(input);
   } catch (error) {
     if (error instanceof ZodError) {
       const message = error.issues.map((e: z.ZodIssue) => e.message).join(", ");
@@ -712,15 +717,30 @@ function latestArtifactRedirect(kind: OtaKind) {
 
 export const RetrieveLatestApp = latestArtifactRedirect("app");
 
+const syncBodySchema = z
+  .object({
+    type: z.string().refine(type => OTA_PREFIXES.includes(type), "Unknown release type"),
+    version: z.string().min(1),
+  })
+  .partial()
+  .refine(
+    body => (body.type === undefined) === (body.version === undefined),
+    "type and version go together",
+  )
+  .transform(body =>
+    body.type && body.version ? { type: body.type, version: body.version } : undefined,
+  );
+
 /**
  * POST /releases/sync: register every stable R2 version missing from the DB
- * and answer with the per-outcome counts. Made for the upload script's last
- * step, so the settle window is off: the caller vouches its last object is
- * written.
+ * and answer with the per-outcome counts. With `{ type, version }` in the
+ * body, that one version skips the settle window: the caller vouches its
+ * last object is written. Every other version keeps it.
  */
 export function Sync(runner: ReleaseSyncRunner) {
   return async (req: Request, res: Response) => {
-    const stats = await runner({ uploadSettleMs: 0 });
+    const settled = parseOrBadRequest(syncBodySchema, req.body ?? {});
+    const stats = await runner({ settled });
     if (stats === "busy") {
       throw new ConflictError("A release sync is already in progress");
     }
diff --git a/test/sync-releases.test.ts b/test/sync-releases.test.ts
index fd55f52..3d340d4 100644
--- a/test/sync-releases.test.ts
+++ b/test/sync-releases.test.ts
@@ -19,7 +19,7 @@ import {
   type ReleaseType,
 } from "../src/release-sync";
 import { Sync } from "../src/releases";
-import { ConflictError } from "../src/errors";
+import { BadRequestError, ConflictError } from "../src/errors";
 import { otaFileForPrefix } from "../src/skus";
 
 import { createAsyncIterable, s3Mock, testPrisma } from "./setup";
@@ -520,31 +520,68 @@ describe("scheduleReleaseSync", () => {
 });
 
 describe("Sync handler", () => {
-  const request = {} as Request;
+  const FRESH_VERSION = "9.9.12";
+  const OTHER_FRESH_VERSION = "9.9.13";
+
+  function request(body?: unknown): Request {
+    return { body } as Request;
+  }
+
+  function response(): Response {
+    return { json: vi.fn() } as unknown as Response;
+  }
+
+  /** Two versions inside the settle window; the scheduled tick would defer both. */
+  function mockTwoFreshVersions() {
+    mockS3ListVersions("app", [FRESH_VERSION, OTHER_FRESH_VERSION]);
+    for (const version of [FRESH_VERSION, OTHER_FRESH_VERSION]) {
+      mockS3HashFile("app", version, `${version}-hash`);
+      mockS3UploadedAt("app", version, new Date(Date.now() - 60 * 1000));
+    }
+  }
 
-  it("registers a version uploaded moments ago and answers with the counts", async () => {
-    const version = "9.9.12";
-    mockS3ListVersions("app", [version]);
-    mockS3HashFile("app", version, "fresh-hash");
-    // Inside the settle window: the scheduled tick would defer this version,
-    // the endpoint trusts the caller and registers it.
-    mockS3UploadedAt("app", version, new Date(Date.now() - 60 * 1000));
-    const res = { json: vi.fn() } as unknown as Response;
+  it("registers only the vouched version and defers the other fresh one", async () => {
+    mockTwoFreshVersions();
+    const res = response();
 
-    await Sync(newRunner())(request, res);
+    await Sync(newRunner())(request({ type: "app", version: FRESH_VERSION }), res);
 
-    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ created: 1, uploading: 0 }));
-    const release = await testPrisma.release.findUniqueOrThrow({
-      where: { version_type: { version, type: "app" } },
-    });
-    expect(release.rolloutPercentage).toBe(10);
+    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ created: 1, uploading: 1 }));
+    expect(
+      await testPrisma.release.findUnique({
+        where: { version_type: { version: FRESH_VERSION, type: "app" } },
+      }),
+    ).toMatchObject({ rolloutPercentage: 10 });
+    expect(
+      await testPrisma.release.findUnique({
+        where: { version_type: { version: OTHER_FRESH_VERSION, type: "app" } },
+      }),
+    ).toBeNull();
+  });
+
+  it("keeps the settle window for every version without a body", async () => {
+    mockTwoFreshVersions();
+    const res = response();
+
+    await Sync(newRunner())(request(), res);
+
+    expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ created: 0, uploading: 2 }));
+  });
+
+  it.each([
+    ["type without version", { type: "app" }],
+    ["version without type", { version: "1.0.0" }],
+    ["unknown type", { type: "firmware", version: "1.0.0" }],
+  ])("rejects %s without running", async (_label, body) => {
+    const run = vi.fn();
+
+    await expect(Sync(run)(request(body), response())).rejects.toThrow(BadRequestError);
+    expect(run).not.toHaveBeenCalled();
   });
 
   it("answers 409 while a run is already in progress", async () => {
     const busyRunner = vi.fn().mockResolvedValue("busy");
 
-    await expect(Sync(busyRunner)(request, { json: vi.fn() } as unknown as Response)).rejects.toThrow(
-      ConflictError,
-    );
+    await expect(Sync(busyRunner)(request(), response())).rejects.toThrow(ConflictError);
   });
 });

From 15879108bae54f327f8e8b117a5238a8985d0a1d Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Mon, 21 Sep 2026 15:49:51 +0200
Subject: [PATCH 07/12] fix(releases): no compatible default is null, not 404
 (#88)

getDefaultRelease threw when 100% releases existed for the prefix but
none shipped a binary for the SKU. Retrieve awaits that fallback before
it checks whether the device is in the staged release's bucket, so the
first staged release for a new SKU on a shared prefix answered 404 to
every device of that SKU. It now returns null, like a prefix with no
100% release at all, and the caller's bucket check decides.
---
 src/releases.ts       | 10 +++++-----
 test/releases.test.ts | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/releases.ts b/src/releases.ts
index 30310fd..49c48c5 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -428,8 +428,10 @@ function dbReleaseToMetadata(release: DbRelease, sku: string): ReleaseMetadata {
 }
 
 /**
- * Newest fully rolled out release for the prefix, or null when no release
- * has reached 100% yet (a prefix whose first release is still staged).
+ * Newest fully rolled out release that ships a binary for the SKU, or null
+ * when there is none: the prefix's first release is still staged, or the SKU
+ * is new and its first build has not reached 100% yet. The caller decides
+ * whether the device is in the staged release's bucket before it needs this.
  */
 async function getDefaultRelease(prefix: string, sku: string): Promise<DbRelease | null> {
   const rolledOutReleases = await prisma.release.findMany({
@@ -447,9 +449,7 @@ async function getDefaultRelease(prefix: string, sku: string): Promise<DbRelease
   const compatibleReleases = rolledOutReleases.filter(r => r.artifacts.length > 0);
 
   if (compatibleReleases.length === 0) {
-    throw new NotFoundError(
-      `No default ${prefix} release available for SKU "${sku}"`,
-    );
+    return null;
   }
 
   const latestVersion = semver.maxSatisfying(
diff --git a/test/releases.test.ts b/test/releases.test.ts
index 9d672bc..2f0c189 100644
--- a/test/releases.test.ts
+++ b/test/releases.test.ts
@@ -608,6 +608,21 @@ describe("Retrieve handler", () => {
         Retrieve(createMockRequest({ deviceId: "late-adopter" }), createMockResponse()),
       ).rejects.toThrow(/No (app|system) release is rolled out yet for SKU "jetkvm-v2"/);
     });
+
+    it("serves the first staged release for a new SKU when no 100% release ships it", async () => {
+      // The seeded 100% releases carry jetkvm-v2 only. The first SDMMC build
+      // is staged at 50%: early-adopter (bucket 8) is in, late-adopter (95) out.
+      await createDbRelease("app", "1.3.0", 50, [releaseArtifact("app", "1.3.0", SDMMC_SKU)]);
+      await createDbRelease("system", "1.3.0", 50, [releaseArtifact("system", "1.3.0", SDMMC_SKU)]);
+
+      const inBucket = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "early-adopter", sku: SDMMC_SKU }), inBucket);
+      expect(jsonBody(inBucket)).toMatchObject({ appVersion: "1.3.0", systemVersion: "1.3.0" });
+
+      await expect(
+        Retrieve(createMockRequest({ deviceId: "late-adopter", sku: SDMMC_SKU }), createMockResponse()),
+      ).rejects.toThrow(/No (app|system) release is rolled out yet for SKU "jetkvm-v2-sdmmc"/);
+    });
   });
 
   describe("signature URL handling", () => {

From 90fcf63335f4232869295b4a49acd65dcf38b648 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Mon, 21 Sep 2026 15:55:01 +0200
Subject: [PATCH 08/12] Releases: register new releases at 0% rollout (#89)

* feat(releases): register new releases at 0% rollout

A newly registered release went to 10% of devices at once. It now
starts at 0%: registered, but served to no device until an operator
raises it. Existing rows keep their percentage.

* fix(releases): a 0% release does not displace a rollout in progress

Selection took the single newest release as the rollout candidate, so
registering the next version at 0% pushed every device off an older
release still mid-rollout and onto the last 100% one. The latest lookup
now ignores rows at 0%. Version-constrained requests, which skip
rollout, still see them.
---
 src/release-sync.ts        |  3 ++-
 src/releases.ts            | 10 ++++++++--
 test/releases.test.ts      | 15 +++++++++++++++
 test/sync-releases.test.ts | 10 +++++-----
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/src/release-sync.ts b/src/release-sync.ts
index b176040..8fdd32d 100644
--- a/src/release-sync.ts
+++ b/src/release-sync.ts
@@ -36,7 +36,8 @@ export interface ReleaseArtifactInput {
   compatibleSkus: string[];
 }
 
-export const DEFAULT_ROLLOUT_PERCENTAGE = 10;
+/** A new release is registered but served to no device until an operator raises it. */
+export const DEFAULT_ROLLOUT_PERCENTAGE = 0;
 
 export type ReleaseOutcome =
   | "created"
diff --git a/src/releases.ts b/src/releases.ts
index 49c48c5..bc10035 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -1,4 +1,5 @@
 import { Request, Response } from "express";
+import type { Prisma } from "@prisma/client";
 import { prisma } from "./db";
 import { BadRequestError, ConflictError, InternalServerError, NotFoundError } from "./errors";
 import type { ReleaseSyncRunner } from "./release-sync";
@@ -468,17 +469,22 @@ async function getDefaultRelease(prefix: string, sku: string): Promise<DbRelease
   return latestDefaultRelease;
 }
 
+/**
+ * Newest release that is rolling out at all. A row at 0% is registered but
+ * not yet released, so it must not displace an older release mid-rollout.
+ */
 async function getLatestRelease(prefix: string, sku: string): Promise<DbRelease> {
-  return getReleaseByRange(prefix, sku, "*");
+  return getReleaseByRange(prefix, sku, "*", { rolloutPercentage: { gt: 0 } });
 }
 
 async function getReleaseByRange(
   prefix: string,
   sku: string,
   range: string,
+  where: Prisma.ReleaseWhereInput = {},
 ): Promise<DbRelease> {
   const releases = await prisma.release.findMany({
-    where: { type: prefix },
+    where: { type: prefix, ...where },
     select: compatibleReleaseSelect(sku),
   });
 
diff --git a/test/releases.test.ts b/test/releases.test.ts
index 2f0c189..7d18033 100644
--- a/test/releases.test.ts
+++ b/test/releases.test.ts
@@ -392,6 +392,21 @@ describe("Retrieve handler", () => {
       });
     });
 
+    it("keeps an in-progress rollout when a newer release is registered at 0%", async () => {
+      // early-adopter hashes to bucket 8, late-adopter to bucket 95.
+      await createDbReleasePair("2.0.0", 100);
+      await createDbReleasePair("2.1.0", 50);
+      await createDbReleasePair("2.2.0", 0);
+
+      const inBucket = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "early-adopter" }), inBucket);
+      expect(jsonBody(inBucket)).toMatchObject({ appVersion: "2.1.0", systemVersion: "2.1.0" });
+
+      const outOfBucket = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "late-adopter" }), outOfBucket);
+      expect(jsonBody(outOfBucket)).toMatchObject({ appVersion: "2.0.0", systemVersion: "2.0.0" });
+    });
+
     it("applies app and system rollout independently", async () => {
       await createDbReleasePair("2.4.0", 100);
       await createDbRelease("app", "2.5.0", 100);
diff --git a/test/sync-releases.test.ts b/test/sync-releases.test.ts
index 3d340d4..87e081b 100644
--- a/test/sync-releases.test.ts
+++ b/test/sync-releases.test.ts
@@ -201,7 +201,7 @@ describe("syncReleases", () => {
     expect(artifacts).toEqual([]);
   });
 
-  it("creates new releases at 10% with their S3 artifacts and skips already-synced versions", async () => {
+  it("creates new releases at 0% with their S3 artifacts and skips already-synced versions", async () => {
     const version = "9.9.4";
 
     // Pre-existing system row simulates a release the migration (or a prior
@@ -249,8 +249,8 @@ describe("syncReleases", () => {
       where: { version_type: { version: "10.0.0-beta.1", type: "app" } },
     });
 
-    // App release is new — created at 10% rollout with a single legacy-compatible artifact.
-    expect(appRelease.rolloutPercentage).toBe(10);
+    // App release is new — created at 0% rollout with a single legacy-compatible artifact.
+    expect(appRelease.rolloutPercentage).toBe(0);
     expect(appRelease.artifacts).toEqual([
       expect.objectContaining({
         url: `https://cdn.test.com/app/${version}/jetkvm_app`,
@@ -480,7 +480,7 @@ describe("scheduleReleaseSync", () => {
       const release = await testPrisma.release.findUnique({
         where: { version_type: { version, type: "app" } },
       });
-      expect(release?.rolloutPercentage).toBe(10);
+      expect(release?.rolloutPercentage).toBe(0);
     });
   });
 
@@ -551,7 +551,7 @@ describe("Sync handler", () => {
       await testPrisma.release.findUnique({
         where: { version_type: { version: FRESH_VERSION, type: "app" } },
       }),
-    ).toMatchObject({ rolloutPercentage: 10 });
+    ).toMatchObject({ rolloutPercentage: 0 });
     expect(
       await testPrisma.release.findUnique({
         where: { version_type: { version: OTHER_FRESH_VERSION, type: "app" } },

From 4e7c46228658ad4c5c0bbbcf67ea7ad4f5ba855e Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Wed, 21 Oct 2026 17:07:46 +0200
Subject: [PATCH 09/12] Require Node.js 22.23.4 or later for the October 2026
 security release

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 4cdeb56..8374041 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
     "test:coverage": "vitest run --coverage"
   },
   "engines": {
-    "node": "22.x"
+    "node": ">=22.23.4 <23"
   },
   "keywords": [],
   "author": "JetKVM",

From d08418e650b6a03c17dbdc90bbd7cde2b105c7f2 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Wed, 21 Oct 2026 17:42:32 +0200
Subject: [PATCH 10/12] Update the Dockerfile to Node.js 22.23.4 and run like
 production

Run the TypeScript source with ts-node, as the App Platform deploy does.
Migrations are not part of the image build; run them separately.
---
 .dockerignore |  1 +
 Dockerfile    | 32 ++++++++++++--------------------
 2 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index e4d319e..33d703b 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -4,3 +4,4 @@ node_modules/
 dist/
 .DS_Store
 .env
+.git
diff --git a/Dockerfile b/Dockerfile
index 3825248..ad38c46 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,29 +1,21 @@
-FROM node:21.1.0-alpine AS packages
+FROM node:22.23.4-bookworm-slim
+LABEL org.opencontainers.image.source="https://github.com/jetkvm/cloud-api"
 WORKDIR /usr/src/app
 
-COPY LICENSE /usr/src/app/
-COPY package.json /usr/src/app/
-COPY package-lock.json /usr/src/app/
-
-RUN npm install
+# Prisma needs OpenSSL at runtime
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends openssl ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
 
-FROM packages AS builder
-WORKDIR /usr/src/app
+COPY package.json package-lock.json ./
+COPY prisma ./prisma
+# ts-node type-checks at startup, so the @types/* devDependencies are needed at runtime
+RUN npm ci
 
-COPY . /usr/src/app/
+COPY . .
 RUN npx prisma generate
-RUN npm run build
-
-FROM packages AS app
-LABEL org.opencontainers.image.source="https://github.com/jetkvm/cloud-api"
-WORKDIR /usr/src/app
-
-COPY --from=builder /usr/src/app/prisma /usr/src/app/prisma
-COPY --from=builder /usr/src/app/node_modules/.prisma /usr/src/app/node_modules/.prisma
-COPY --from=builder /usr/src/app/dist /usr/src/app/dist
-COPY .env.example ./.env
 
 ENV NODE_ENV=production
 ENV PORT=3000
 EXPOSE 3000
-CMD ["node", "./dist/index.js"]
+CMD ["npm", "run", "start"]

From 5f951afdba0d4923402c3e08fe9064e3c671fbd3 Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Fri, 6 Nov 2026 10:43:57 +0100
Subject: [PATCH 11/12] Build the image in stages and run compiled JavaScript
 (#92)

Compile with tsc at build time, so a type error fails the build instead
of crash-looping the container. The runtime stage has production
dependencies only, runs as the node user under tini, and has a
HEALTHCHECK on /healthz.

Move ts-node, typescript, @tsconfig/node22 and @types/* to
devDependencies. Keep .env.* files out of the build context. Run CI on
Node.js 22.23.4 to match engines.
---
 .dockerignore                      |  2 +
 .github/workflows/pull-request.yml |  2 +-
 Dockerfile                         | 29 +++++++++----
 package-lock.json                  | 68 ++++++++++++++++++++++--------
 package.json                       | 18 ++++----
 5 files changed, 83 insertions(+), 36 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 33d703b..9c432ec 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,5 @@ dist/
 .DS_Store
 .env
 .git
+.env.*
+!.env.example
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index 12fe432..dd7abdf 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -31,7 +31,7 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v4
       with:
-        node-version: v22.21.0
+        node-version: v22.23.4
         cache: 'npm'
         cache-dependency-path: '**/package-lock.json'
 
diff --git a/Dockerfile b/Dockerfile
index ad38c46..33ff589 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,21 +1,34 @@
-FROM node:22.23.4-bookworm-slim
+FROM node:22.23.4-bookworm-slim AS base
 LABEL org.opencontainers.image.source="https://github.com/jetkvm/cloud-api"
 WORKDIR /usr/src/app
 
-# Prisma needs OpenSSL at runtime
+# Prisma needs OpenSSL at runtime; tini forwards SIGTERM to node as PID 1
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends openssl ca-certificates \
+  && apt-get install -y --no-install-recommends openssl ca-certificates tini \
   && rm -rf /var/lib/apt/lists/*
 
+FROM base AS build
 COPY package.json package-lock.json ./
 COPY prisma ./prisma
-# ts-node type-checks at startup, so the @types/* devDependencies are needed at runtime
 RUN npm ci
+COPY tsconfig.json ./
+COPY src ./src
+# Type errors fail the build here instead of crashing the container at startup
+RUN npx prisma generate \
+  && npm run build \
+  && npm prune --omit=dev
 
-COPY . .
-RUN npx prisma generate
-
+FROM base
 ENV NODE_ENV=production
 ENV PORT=3000
+COPY package.json ./
+COPY prisma ./prisma
+COPY --from=build /usr/src/app/node_modules ./node_modules
+COPY --from=build /usr/src/app/dist ./dist
+
+USER node
 EXPOSE 3000
-CMD ["npm", "run", "start"]
+HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
+  CMD ["node", "-e", "fetch('http://localhost:' + process.env.PORT + '/healthz').then(r => process.exit(r.ok ? 0 : 1), () => process.exit(1))"]
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["node", "dist/index.js"]
diff --git a/package-lock.json b/package-lock.json
index 8579101..fe9214e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,11 +11,6 @@
       "dependencies": {
         "@aws-sdk/client-s3": "^3.654.0",
         "@prisma/client": "^5.13.0",
-        "@tsconfig/node22": "^22.0.0",
-        "@types/cookie-session": "^2.0.49",
-        "@types/cors": "^2.8.17",
-        "@types/node": "^20.12.10",
-        "@types/ws": "^8.5.10",
         "cookie-session": "^2.1.0",
         "cors": "^2.8.5",
         "dotenv": "^16.4.7",
@@ -27,22 +22,27 @@
         "openid-client": "^5.6.5",
         "prisma": "^5.13.0",
         "semver": "^7.6.3",
-        "ts-node": "^10.9.2",
-        "typescript": "^5.4.5",
         "ws": "^8.17.1",
         "zod": "^4.3.6"
       },
       "devDependencies": {
+        "@tsconfig/node22": "^22.0.0",
+        "@types/cookie-session": "^2.0.49",
+        "@types/cors": "^2.8.17",
         "@types/express": "^5.0.6",
         "@types/lru-cache": "^7.10.9",
+        "@types/node": "^20.12.10",
         "@types/semver": "^7.5.8",
+        "@types/ws": "^8.5.10",
         "@vitest/coverage-v8": "^4.0.18",
         "aws-sdk-client-mock": "^4.1.0",
         "prettier": "3.2.5",
+        "ts-node": "^10.9.2",
+        "typescript": "^5.4.5",
         "vitest": "^4.0.18"
       },
       "engines": {
-        "node": "22.21.0"
+        "node": ">=22.23.4 <23"
       },
       "optionalDependencies": {
         "bufferutil": "^4.0.8"
@@ -964,6 +964,7 @@
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
       "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
+      "dev": true,
       "dependencies": {
         "@jridgewell/trace-mapping": "0.3.9"
       },
@@ -1417,6 +1418,7 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
       "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
+      "dev": true,
       "engines": {
         "node": ">=6.0.0"
       }
@@ -1425,12 +1427,14 @@
       "version": "1.5.5",
       "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
       "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/@jridgewell/trace-mapping": {
       "version": "0.3.9",
       "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
       "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
+      "dev": true,
       "dependencies": {
         "@jridgewell/resolve-uri": "^3.0.3",
         "@jridgewell/sourcemap-codec": "^1.4.10"
@@ -2541,32 +2545,38 @@
     "node_modules/@tsconfig/node10": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz",
-      "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw=="
+      "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==",
+      "dev": true
     },
     "node_modules/@tsconfig/node12": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
-      "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag=="
+      "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
+      "dev": true
     },
     "node_modules/@tsconfig/node14": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
-      "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow=="
+      "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
+      "dev": true
     },
     "node_modules/@tsconfig/node16": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
-      "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA=="
+      "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
+      "dev": true
     },
     "node_modules/@tsconfig/node22": {
       "version": "22.0.0",
       "resolved": "https://registry.npmjs.org/@tsconfig/node22/-/node22-22.0.0.tgz",
-      "integrity": "sha512-twLQ77zevtxobBOD4ToAtVmuYrpeYUh3qh+TEp+08IWhpsrIflVHqQ1F1CiPxQGL7doCdBIOOCF+1Tm833faNg=="
+      "integrity": "sha512-twLQ77zevtxobBOD4ToAtVmuYrpeYUh3qh+TEp+08IWhpsrIflVHqQ1F1CiPxQGL7doCdBIOOCF+1Tm833faNg==",
+      "dev": true
     },
     "node_modules/@types/body-parser": {
       "version": "1.19.5",
       "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz",
       "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==",
+      "dev": true,
       "dependencies": {
         "@types/connect": "*",
         "@types/node": "*"
@@ -2587,6 +2597,7 @@
       "version": "3.4.38",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
       "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==",
+      "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
@@ -2595,6 +2606,7 @@
       "version": "2.0.49",
       "resolved": "https://registry.npmjs.org/@types/cookie-session/-/cookie-session-2.0.49.tgz",
       "integrity": "sha512-4E/bBjlqLhU5l4iGPR+NkVJH593hpNsT4dC3DJDr+ODm6Qpe13kZQVkezRIb+TYDXaBMemS3yLQ+0leba3jlkQ==",
+      "dev": true,
       "dependencies": {
         "@types/express": "*",
         "@types/keygrip": "*"
@@ -2604,6 +2616,7 @@
       "version": "2.8.17",
       "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz",
       "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==",
+      "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
@@ -2626,6 +2639,7 @@
       "version": "5.0.6",
       "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz",
       "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==",
+      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/body-parser": "*",
@@ -2637,6 +2651,7 @@
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz",
       "integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==",
+      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/node": "*",
@@ -2649,6 +2664,7 @@
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz",
       "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/http-proxy": {
@@ -2662,7 +2678,8 @@
     "node_modules/@types/keygrip": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.6.tgz",
-      "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ=="
+      "integrity": "sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==",
+      "dev": true
     },
     "node_modules/@types/lru-cache": {
       "version": "7.10.9",
@@ -2686,12 +2703,14 @@
       "version": "6.14.0",
       "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz",
       "integrity": "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/range-parser": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz",
       "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/@types/semver": {
@@ -2704,6 +2723,7 @@
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz",
       "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==",
+      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/node": "*"
@@ -2713,6 +2733,7 @@
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz",
       "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==",
+      "dev": true,
       "license": "MIT",
       "dependencies": {
         "@types/http-errors": "*",
@@ -2740,6 +2761,7 @@
       "version": "8.5.10",
       "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz",
       "integrity": "sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==",
+      "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
@@ -2875,6 +2897,7 @@
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
       "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
+      "dev": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -2886,6 +2909,7 @@
       "version": "8.3.2",
       "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz",
       "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==",
+      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -2893,7 +2917,8 @@
     "node_modules/arg": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
-      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="
+      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+      "dev": true
     },
     "node_modules/assertion-error": {
       "version": "2.0.1",
@@ -3157,7 +3182,8 @@
     "node_modules/create-require": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
-      "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ=="
+      "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
+      "dev": true
     },
     "node_modules/debug": {
       "version": "4.3.6",
@@ -3196,6 +3222,7 @@
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
       "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "dev": true,
       "engines": {
         "node": ">=0.3.1"
       }
@@ -3865,7 +3892,8 @@
     "node_modules/make-error": {
       "version": "1.3.6",
       "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
-      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw=="
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
+      "dev": true
     },
     "node_modules/math-intrinsics": {
       "version": "1.1.0",
@@ -4689,6 +4717,7 @@
       "version": "10.9.2",
       "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
       "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
+      "dev": true,
       "dependencies": {
         "@cspotcode/source-map-support": "^0.8.0",
         "@tsconfig/node10": "^1.0.7",
@@ -4767,6 +4796,7 @@
       "version": "5.4.5",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
       "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
+      "dev": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -4811,7 +4841,8 @@
     "node_modules/v8-compile-cache-lib": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
-      "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="
+      "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
+      "dev": true
     },
     "node_modules/vary": {
       "version": "1.1.2",
@@ -5084,6 +5115,7 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
       "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
+      "dev": true,
       "engines": {
         "node": ">=6"
       }
diff --git a/package.json b/package.json
index 8374041..2ddec3a 100644
--- a/package.json
+++ b/package.json
@@ -2,9 +2,9 @@
   "name": "jetkvm-cloud-api",
   "version": "1.0.0",
   "description": "JetKVM Cloud API and Websocket Server",
-  "main": "dist/src/index.js",
+  "main": "dist/index.js",
   "scripts": {
-    "start": "NODE_ENV=production node -r ts-node/register ./src/index.ts",
+    "start": "NODE_ENV=production node dist/index.js",
     "dev": "NODE_ENV=development node --watch --watch-path=./src --env-file=.env.development -r ts-node/register ./src/index.ts",
     "dev:debug": "NODE_ENV=development node --watch --watch-path=./src --env-file=.env.development -r ts-node/register ./src/index.ts --inspect-brk",
     "prisma-dev": "prisma generate --watch",
@@ -27,11 +27,6 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.654.0",
     "@prisma/client": "^5.13.0",
-    "@tsconfig/node22": "^22.0.0",
-    "@types/cookie-session": "^2.0.49",
-    "@types/cors": "^2.8.17",
-    "@types/node": "^20.12.10",
-    "@types/ws": "^8.5.10",
     "cookie-session": "^2.1.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.7",
@@ -43,8 +38,6 @@
     "openid-client": "^5.6.5",
     "prisma": "^5.13.0",
     "semver": "^7.6.3",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.4.5",
     "ws": "^8.17.1",
     "zod": "^4.3.6"
   },
@@ -52,12 +45,19 @@
     "bufferutil": "^4.0.8"
   },
   "devDependencies": {
+    "@tsconfig/node22": "^22.0.0",
+    "@types/cookie-session": "^2.0.49",
+    "@types/cors": "^2.8.17",
     "@types/express": "^5.0.6",
     "@types/lru-cache": "^7.10.9",
+    "@types/node": "^20.12.10",
     "@types/semver": "^7.5.8",
+    "@types/ws": "^8.5.10",
     "@vitest/coverage-v8": "^4.0.18",
     "aws-sdk-client-mock": "^4.1.0",
     "prettier": "3.2.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.5",
     "vitest": "^4.0.18"
   }
 }

From 0f95ac282fc849d6c0b75100177cd63aa7da40aa Mon Sep 17 00:00:00 2001
From: Adam Shiervani <adam.shiervani@gmail.com>
Date: Fri, 6 Nov 2026 13:03:30 +0100
Subject: [PATCH 12/12] Add end-to-end tests and a deployment compare script
 (#93)

npm run test:e2e builds the API and runs dist/index.js against Postgres and
SeaweedFS (S3): release registration and update checks against real files and
hashes, rollouts from upload to 100%, a new SKU's first staged release, auth,
CORS, headers, device registration and removal, and WebSocket signaling. CI
runs them after the unit tests.

npm run compare sends the same safe requests to two deployments, groups the
differences by cause and, in a terminal, asks whether each group is expected.
It replaces scripts/compare-releases.sh.

DELETE /devices/:id without credentials answers 401. The WebSocket upgrade
handler and device disconnect cleanup catch their errors.

With appVersion or systemVersion, only that part is pinned; the other follows
the rollout. Prerelease update checks fall back to the newest of the five
newest versions with a build for the SKU. The latest-file redirects answer 404
when the newest version has no file for the SKU.
---
 .github/workflows/pull-request.yml |   9 +
 README.md                          |  25 +-
 compose.development.yaml           |   8 +
 e2e/devices.test.ts                | 125 +++++
 e2e/fixtures.ts                    |  75 +++
 e2e/global-setup.ts                | 234 ++++++++
 e2e/helpers.ts                     | 175 ++++++
 e2e/http.test.ts                   | 136 +++++
 e2e/release-sync.test.ts           | 204 +++++++
 e2e/releases.test.ts               | 320 +++++++++++
 e2e/signaling.test.ts              | 229 ++++++++
 e2e/support.ts                     |  44 ++
 package.json                       |   4 +-
 scripts/compare-api.mts            | 538 ++++++++++++++++++
 scripts/compare-releases.sh        | 875 -----------------------------
 src/auth.ts                        |   8 +-
 src/devices.ts                     |  17 +-
 src/releases.ts                    | 269 ++++-----
 src/webrtc-signaling.ts            | 113 ++--
 test/releases.test.ts              |  72 ++-
 vitest.e2e.config.ts               |  13 +
 21 files changed, 2389 insertions(+), 1104 deletions(-)
 create mode 100644 e2e/devices.test.ts
 create mode 100644 e2e/fixtures.ts
 create mode 100644 e2e/global-setup.ts
 create mode 100644 e2e/helpers.ts
 create mode 100644 e2e/http.test.ts
 create mode 100644 e2e/release-sync.test.ts
 create mode 100644 e2e/releases.test.ts
 create mode 100644 e2e/signaling.test.ts
 create mode 100644 e2e/support.ts
 create mode 100644 scripts/compare-api.mts
 delete mode 100755 scripts/compare-releases.sh
 create mode 100644 vitest.e2e.config.ts

diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index dd7abdf..cd37dbe 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -28,6 +28,10 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    # Pulls and starts in the background while dependencies install; the end-to-end tests use it
+    - name: Start S3-compatible store
+      run: nohup docker compose -f compose.development.yaml up -d s3 > "$RUNNER_TEMP/s3-start.log" 2>&1 &
+
     - name: Use Node.js
       uses: actions/setup-node@v4
       with:
@@ -57,6 +61,11 @@ jobs:
     - name: Run Tests
       run: npm test
 
+    - name: Run end-to-end tests
+      run: |
+        timeout 180 sh -c 'until curl -s -o /dev/null http://127.0.0.1:8333; do sleep 1; done' || { cat "$RUNNER_TEMP/s3-start.log"; exit 1; }
+        npx vitest run --config vitest.e2e.config.ts
+
     - name: Cache Prisma Binary
       uses: actions/cache@v4
       with:
diff --git a/README.md b/README.md
index 0c431eb..d8b7acd 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ If you've found an issue and want to report it, please check our [Issues](https:
 This project is built with Node.js, Prisma, and Express.
 
 ```bash
-# Start the database
+# Start the database and the S3-compatible store the e2e tests use
 docker compose -f compose.development.yaml up -d
 
 # Copy and configure environment variables
@@ -50,6 +50,29 @@ npm run dev
 npm test
 ```
 
+### End-to-end tests
+
+`npm run test:e2e` builds the API and starts it as the container does, against a
+database (`jetkvm_e2e`) and a bucket that only the e2e run uses. It needs the
+services from `compose.development.yaml`. Override them with `E2E_DATABASE_URL`
+and `E2E_S3_ENDPOINT`.
+
+### Comparing two deployments
+
+`npm run compare` sends the same requests to two deployments of the API and
+reports where the answers differ, grouped by cause. In a terminal it asks you to
+mark each group as expected or not, and exits with 0 only when you mark them all:
+
+```bash
+npm run compare -- --candidate local
+npm run compare -- --candidate staging --only releases,redirects
+npm run compare -- --help
+```
+
+The reference defaults to `https://api.jetkvm.com`. Update checks that depend
+on the rollout table differ unless both databases have the same rollout
+percentages.
+
 ## Self-Hosting
 
 For self-hosting, use the default compose file which runs the complete stack:
diff --git a/compose.development.yaml b/compose.development.yaml
index 5c601a4..ee0229f 100644
--- a/compose.development.yaml
+++ b/compose.development.yaml
@@ -19,6 +19,14 @@ services:
     volumes:
       - postgresql:/var/lib/postgresql
 
+  # S3-compatible store for the e2e tests (npm run test:e2e)
+  s3:
+    image: chrislusf/seaweedfs:4.50
+    restart: unless-stopped
+    command: server -s3 -dir=/data
+    ports:
+      - "8333:8333"
+
 volumes:
   postgresql:
     driver: local
diff --git a/e2e/devices.test.ts b/e2e/devices.test.ts
new file mode 100644
index 0000000..0bb2868
--- /dev/null
+++ b/e2e/devices.test.ts
@@ -0,0 +1,125 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import {
+  api,
+  closed,
+  createUserWithDevice,
+  db,
+  openSocket,
+  sessionCookie,
+  unsignedIdToken,
+} from "./helpers";
+
+beforeEach(async () => {
+  await db.device.deleteMany({ where: { id: { startsWith: "e2e-dev-" } } });
+  await db.user.deleteMany({ where: { googleId: { startsWith: "e2e-dev-" } } });
+});
+
+function exchange(tempToken?: string) {
+  return api("/devices/token", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(tempToken === undefined ? {} : { tempToken }),
+  });
+}
+
+async function deviceWithTempToken(id: string, tempToken: string, expiresInMs: number) {
+  await db.user.create({
+    data: {
+      googleId: `e2e-dev-${id}-owner`,
+      device: {
+        create: { id, tempToken, tempTokenExpiresAt: new Date(Date.now() + expiresInMs) },
+      },
+    },
+  });
+}
+
+describe("device registration (POST /devices/token)", () => {
+  it("trades a temporary token for a secret token the device can connect with, once", async () => {
+    await deviceWithTempToken("e2e-dev-1", "temp-1", 60_000);
+
+    const response = await exchange("temp-1");
+    expect(response.status).toBe(200);
+    const { secretToken } = await response.json();
+    expect(secretToken).toMatch(/^[0-9a-f]{40}$/);
+
+    const device = await db.device.findUniqueOrThrow({ where: { id: "e2e-dev-1" } });
+    expect(device).toMatchObject({
+      secretToken,
+      tempToken: null,
+      tempTokenExpiresAt: null,
+    });
+
+    // The temporary token is single-use
+    expect((await exchange("temp-1")).status).toBe(404);
+
+    const ws = await openSocket("/", {
+      authorization: `Bearer ${secretToken}`,
+      "x-device-id": "e2e-dev-1",
+    });
+    ws.close();
+    await closed(ws);
+  });
+
+  it("refuses an expired temporary token and leaves the device without a secret", async () => {
+    await deviceWithTempToken("e2e-dev-2", "temp-2", -1_000);
+    expect((await exchange("temp-2")).status).toBe(401);
+    expect(
+      (await db.device.findUniqueOrThrow({ where: { id: "e2e-dev-2" } })).secretToken,
+    ).toBeNull();
+  });
+
+  it("answers 404 to an unknown token and 422 to a missing one", async () => {
+    expect((await exchange("never-issued")).status).toBe(404);
+    expect((await exchange()).status).toBe(422);
+  });
+});
+
+describe("device removal (DELETE /devices/:id)", () => {
+  it("lets the device remove itself with its secret token", async () => {
+    await createUserWithDevice("e2e-dev-owner-3", "e2e-dev-3", "secret-3");
+
+    const response = await api("/devices/e2e-dev-3", {
+      method: "DELETE",
+      headers: { authorization: "Bearer secret-3" },
+    });
+    expect(response.status).toBe(204);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-3" } })).toBeNull();
+  });
+
+  it("answers 404 to a secret token that belongs to no device", async () => {
+    await createUserWithDevice("e2e-dev-owner-4", "e2e-dev-4", "secret-4");
+    const response = await api("/devices/e2e-dev-4", {
+      method: "DELETE",
+      headers: { authorization: "Bearer someone-elses-secret" },
+    });
+    expect(response.status).toBe(404);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-4" } })).not.toBeNull();
+  });
+
+  it("answers 401 without credentials and keeps serving", async () => {
+    await createUserWithDevice("e2e-dev-owner-5", "e2e-dev-5", "secret-5");
+
+    const response = await api("/devices/e2e-dev-5", {
+      method: "DELETE",
+      signal: AbortSignal.timeout(5000),
+    });
+    expect(response.status).toBe(401);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-5" } })).not.toBeNull();
+    expect((await api("/healthz")).status).toBe(200);
+  });
+
+  it("answers 401 to a session whose ID token Google did not sign", async () => {
+    await createUserWithDevice("e2e-dev-owner-6", "e2e-dev-6", "secret-6");
+    const cookie = sessionCookie({
+      id_token: unsignedIdToken({ sub: "e2e-dev-owner-6" }),
+    });
+
+    const response = await api("/devices/e2e-dev-6", {
+      method: "DELETE",
+      headers: { cookie },
+      signal: AbortSignal.timeout(5000),
+    });
+    expect(response.status).toBe(401);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-6" } })).not.toBeNull();
+  });
+});
diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts
new file mode 100644
index 0000000..a63db44
--- /dev/null
+++ b/e2e/fixtures.ts
@@ -0,0 +1,75 @@
+/**
+ * The release bucket the e2e API reads. Global setup uploads every artifact
+ * with a `.sha256` sidecar (and a `.sig` where `sig` is set), as the release
+ * scripts do, then registers each stable version through POST /releases/sync.
+ */
+
+export const CDN_BASE = "https://cdn.e2e.test";
+
+interface FixtureArtifact {
+  key: string;
+  sig?: boolean;
+}
+
+export const ARTIFACTS: FixtureArtifact[] = [
+  // App. 0.3.0 predates the skus/ layout, 0.5.0 has no SDMMC build.
+  { key: "app/0.3.0/jetkvm_app" },
+  { key: "app/0.4.0/skus/jetkvm-v2/jetkvm_app", sig: true },
+  { key: "app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app" },
+  { key: "app/0.5.0/skus/jetkvm-v2/jetkvm_app" },
+  { key: "app/0.6.0-beta.1/skus/jetkvm-v2/jetkvm_app" },
+  { key: "app/0.6.0-beta.1/skus/jetkvm-v2-sdmmc/jetkvm_app" },
+
+  // System, with the recovery images that share its version.
+  { key: "system/0.2.0/system.tar" },
+  { key: "system/0.2.0/update.img" },
+  { key: "system/0.3.0/skus/jetkvm-v2/system.tar" },
+  { key: "system/0.3.0/skus/jetkvm-v2/update.img" },
+  { key: "system/0.3.0/skus/jetkvm-v2-sdmmc/system.tar" },
+  { key: "system/0.3.0/skus/jetkvm-v2-sdmmc/update_sd.img.zip" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2/system.tar" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2/update.img" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2-sdmmc/system.tar" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2-sdmmc/update_sd.img.zip" },
+
+  // Mini. 0.4.0 uses the pre-SKU layout, which no Mini build may use.
+  // 0.5.4 has no wireless build.
+  { key: "mini/0.4.0/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-ethernet/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-ethernet/jetkvm-mini-full.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini-full.bin" },
+  { key: "mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini.bin" },
+  { key: "mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini-full.bin" },
+];
+
+/** Objects that are not releases: a folder whose name is not a version. */
+export const OTHER_OBJECTS = ["app/latest/README.txt"];
+
+export function fixtureBody(key: string): Buffer {
+  return Buffer.from(`e2e fixture ${key}\n`);
+}
+
+/** Every (type, version) folder with no prerelease tag, as the upload script would sync them. */
+export function stableVersions(): { type: string; version: string }[] {
+  const seen = new Map<string, { type: string; version: string }>();
+  for (const { key } of ARTIFACTS) {
+    const [type, version] = key.split("/");
+    if (!version.includes("-")) seen.set(`${type}/${version}`, { type, version });
+  }
+  return [...seen.values()];
+}
+
+/**
+ * Device IDs and their rollout bucket: parseInt(md5(id).slice(0, 8), 16) % 100.
+ * The values are fixed so that a change to the bucketing, which would move
+ * devices between rollout groups mid-rollout, fails the tests.
+ */
+export const DEVICE_IN_BUCKET = {
+  0: "e2e-device-74",
+  9: "e2e-device-167",
+  49: "e2e-device-152",
+  50: "e2e-device-28",
+  81: "e2e-device-9",
+  99: "e2e-device-19",
+} as const;
diff --git a/e2e/global-setup.ts b/e2e/global-setup.ts
new file mode 100644
index 0000000..7ebd4ac
--- /dev/null
+++ b/e2e/global-setup.ts
@@ -0,0 +1,234 @@
+import { spawn, execFile, type ChildProcess } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:net";
+import { promisify } from "node:util";
+import {
+  CreateBucketCommand,
+  DeleteObjectsCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+} from "@aws-sdk/client-s3";
+import { PrismaClient } from "@prisma/client";
+import type { TestProject } from "vitest/node";
+import {
+  ARTIFACTS,
+  CDN_BASE,
+  OTHER_OBJECTS,
+  fixtureBody,
+  stableVersions,
+} from "./fixtures";
+import { postSync, putArtifact, s3Client } from "./support";
+
+/**
+ * Starts the built API (dist/index.js) the way the container does, against a
+ * Postgres database and an S3-compatible bucket that only the e2e run uses.
+ *
+ *   E2E_DATABASE_URL  default postgresql://jetkvm:jetkvm@localhost:5432/jetkvm_e2e
+ *   E2E_S3_ENDPOINT   default http://127.0.0.1:8333 (SeaweedFS from compose.development.yaml)
+ */
+
+const DATABASE_URL =
+  process.env.E2E_DATABASE_URL ?? "postgresql://jetkvm:jetkvm@localhost:5432/jetkvm_e2e";
+// An IP host makes the AWS SDK use path-style requests, which the local S3 server needs.
+const S3_ENDPOINT = process.env.E2E_S3_ENDPOINT ?? "http://127.0.0.1:8333";
+const BUCKET = "jetkvm-e2e";
+const APP_ORIGIN = "https://app.e2e.test";
+
+declare module "vitest" {
+  export interface ProvidedContext {
+    apiUrl: string;
+    appOrigin: string;
+    databaseUrl: string;
+    s3Endpoint: string;
+    bucket: string;
+    cookieSecret: string;
+    syncToken: string;
+    iceServer: string;
+  }
+}
+
+function assertLocal(url: string, what: string) {
+  const { hostname } = new URL(url);
+  if (!["localhost", "127.0.0.1", "::1"].includes(hostname)) {
+    throw new Error(`${what} must point at this machine, got ${hostname}`);
+  }
+}
+
+async function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const server = createServer();
+    server.listen(0, "127.0.0.1", () => {
+      const { port } = server.address() as { port: number };
+      server.close(() => resolve(port));
+    });
+    server.on("error", reject);
+  });
+}
+
+/** A new, empty database with every migration applied. */
+async function prepareDatabase() {
+  const name = new URL(DATABASE_URL).pathname.slice(1);
+  const admin = new URL(DATABASE_URL);
+  admin.pathname = "/postgres";
+
+  const prisma = new PrismaClient({ datasourceUrl: admin.toString() });
+  try {
+    await prisma.$executeRawUnsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
+    await prisma.$executeRawUnsafe(`CREATE DATABASE "${name}"`);
+  } finally {
+    await prisma.$disconnect();
+  }
+
+  await promisify(execFile)("node_modules/.bin/prisma", ["migrate", "deploy"], {
+    env: { ...process.env, DATABASE_URL },
+  });
+}
+
+/** The bucket, emptied and filled with the fixture releases. */
+async function prepareBucket() {
+  const s3 = s3Client(S3_ENDPOINT);
+  try {
+    await s3.send(new CreateBucketCommand({ Bucket: BUCKET }));
+  } catch (error: any) {
+    if (!["BucketAlreadyOwnedByYou", "BucketAlreadyExists"].includes(error.name))
+      throw error;
+  }
+
+  for (;;) {
+    const { Contents = [] } = await s3.send(new ListObjectsV2Command({ Bucket: BUCKET }));
+    if (Contents.length === 0) break;
+    await s3.send(
+      new DeleteObjectsCommand({
+        Bucket: BUCKET,
+        Delete: { Objects: Contents.map(({ Key }) => ({ Key })) },
+      }),
+    );
+  }
+
+  const put = (Key: string, Body: Buffer | string) =>
+    s3.send(new PutObjectCommand({ Bucket: BUCKET, Key, Body }));
+  await Promise.all([
+    ...ARTIFACTS.map(({ key, sig }) =>
+      Promise.all([
+        putArtifact(s3, BUCKET, key, fixtureBody(key)),
+        sig ? put(`${key}.sig`, `signature of ${key}`) : undefined,
+      ]),
+    ),
+    ...OTHER_OBJECTS.map(key => put(key, fixtureBody(key))),
+  ]);
+}
+
+async function waitForHealth(apiUrl: string, server: ChildProcess) {
+  const deadline = Date.now() + 30_000;
+  while (Date.now() < deadline) {
+    if (server.exitCode !== null || server.signalCode !== null) {
+      throw new Error("The API exited during startup");
+    }
+    try {
+      if ((await fetch(`${apiUrl}/healthz`)).ok) return;
+    } catch {
+      // not listening yet
+    }
+    await new Promise(r => setTimeout(r, 200));
+  }
+  throw new Error("The API did not become healthy");
+}
+
+/** Registers every stable fixture version through the upload script's endpoint. */
+async function registerReleases(apiUrl: string, syncToken: string) {
+  for (const settled of stableVersions()) {
+    for (;;) {
+      const response = await postSync(apiUrl, syncToken, settled);
+      // 409 while the startup sync still runs
+      if (response.status === 409) {
+        await new Promise(r => setTimeout(r, 100));
+        continue;
+      }
+      if (!response.ok) {
+        throw new Error(
+          `sync of ${settled.type} ${settled.version} failed: ${response.status}`,
+        );
+      }
+      break;
+    }
+  }
+}
+
+export default async function setup(project: TestProject) {
+  assertLocal(DATABASE_URL, "E2E_DATABASE_URL");
+  assertLocal(S3_ENDPOINT, "E2E_S3_ENDPOINT");
+
+  const cookieSecret = randomBytes(16).toString("hex");
+  const syncToken = randomBytes(16).toString("hex");
+  const iceServer = "stun:stun.e2e.test:3478";
+
+  await Promise.all([prepareDatabase(), prepareBucket()]);
+
+  const port = await freePort();
+  const apiUrl = `http://127.0.0.1:${port}`;
+  const server = spawn(process.execPath, ["dist/index.js"], {
+    env: {
+      PATH: process.env.PATH,
+      // Keep a local .env out of the run
+      DOTENV_CONFIG_PATH: "/dev/null",
+      NODE_ENV: "production",
+      PORT: String(port),
+      DATABASE_URL,
+      API_HOSTNAME: apiUrl,
+      APP_HOSTNAME: APP_ORIGIN,
+      CORS_ORIGINS: APP_ORIGIN,
+      COOKIE_SECRET: cookieSecret,
+      GOOGLE_CLIENT_ID: "e2e-client-id",
+      GOOGLE_CLIENT_SECRET: "e2e-client-secret",
+      CLOUDFLARE_TURN_ID: "e2e",
+      CLOUDFLARE_TURN_TOKEN: "e2e",
+      R2_ENDPOINT: S3_ENDPOINT,
+      R2_ACCESS_KEY_ID: "e2e",
+      R2_SECRET_ACCESS_KEY: "e2e",
+      R2_BUCKET: BUCKET,
+      R2_CDN_URL: CDN_BASE,
+      REAL_IP_HEADER: "x-real-ip",
+      ICE_SERVERS: iceServer,
+      RELEASE_SYNC_TOKEN: syncToken,
+    },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  let output = "";
+  server.stdout!.on("data", chunk => (output += chunk));
+  server.stderr!.on("data", chunk => (output += chunk));
+  const log = () => output.split("\n").slice(-40).join("\n");
+
+  let stopping = false;
+  let crashed = false;
+  server.on("exit", (code, signal) => {
+    if (stopping) return;
+    crashed = true;
+    // Printed now, so the tests that fail next show why
+    console.error(`\nThe API exited (code ${code}, signal ${signal}):\n${log()}\n`);
+  });
+
+  try {
+    await waitForHealth(apiUrl, server);
+    await registerReleases(apiUrl, syncToken);
+  } catch (error) {
+    stopping = true;
+    server.kill();
+    throw new Error(`${(error as Error).message}:\n${log()}`);
+  }
+
+  project.provide("apiUrl", apiUrl);
+  project.provide("appOrigin", APP_ORIGIN);
+  project.provide("databaseUrl", DATABASE_URL);
+  project.provide("s3Endpoint", S3_ENDPOINT);
+  project.provide("bucket", BUCKET);
+  project.provide("cookieSecret", cookieSecret);
+  project.provide("syncToken", syncToken);
+  project.provide("iceServer", iceServer);
+
+  return () => {
+    stopping = true;
+    server.kill("SIGTERM");
+    if (crashed) throw new Error("The API exited during the e2e run");
+  };
+}
diff --git a/e2e/helpers.ts b/e2e/helpers.ts
new file mode 100644
index 0000000..5c2ea12
--- /dev/null
+++ b/e2e/helpers.ts
@@ -0,0 +1,175 @@
+import { createHmac } from "node:crypto";
+import { DeleteObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3";
+import { PrismaClient } from "@prisma/client";
+import { inject } from "vitest";
+import WebSocket from "ws";
+import { CDN_BASE } from "./fixtures";
+import { postSync, putArtifact, s3Client, sha256 } from "./support";
+
+export { sha256 };
+
+const apiUrl = inject("apiUrl");
+const bucket = inject("bucket");
+const s3 = s3Client(inject("s3Endpoint"));
+
+export const db = new PrismaClient({ datasourceUrl: inject("databaseUrl") });
+
+export function api(path: string, init?: RequestInit) {
+  return fetch(`${apiUrl}${path}`, { redirect: "manual", ...init });
+}
+
+export const cdn = (key: string) => `${CDN_BASE}/${key}`;
+
+export async function objectBytes(key: string): Promise<Buffer> {
+  const response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+  return Buffer.from(await response.Body!.transformToByteArray());
+}
+
+export const uploadArtifact = (key: string, body: Buffer) =>
+  putArtifact(s3, bucket, key, body);
+
+export async function removeArtifact(key: string) {
+  await Promise.all(
+    [key, `${key}.sha256`].map(Key =>
+      s3.send(new DeleteObjectCommand({ Bucket: bucket, Key })),
+    ),
+  );
+}
+
+export const syncReleases = (body?: object) =>
+  postSync(apiUrl, inject("syncToken"), body);
+
+/** The URL is on the CDN, and the object behind it has exactly this SHA-256. */
+export async function expectServable(url: string, hash: string) {
+  if (!url.startsWith(`${CDN_BASE}/`)) throw new Error(`${url} is not on ${CDN_BASE}`);
+  const actual = sha256(await objectBytes(url.slice(CDN_BASE.length + 1)));
+  if (actual !== hash)
+    throw new Error(`${url}: hash ${hash} does not match the object (${actual})`);
+}
+
+type Rollouts = Record<string, Record<string, number>>;
+
+/** Sets every release to 0% except the ones listed, like an operator editing the table. */
+export async function setRollouts(rollouts: Rollouts) {
+  await db.release.updateMany({ data: { rolloutPercentage: 0 } });
+  for (const [type, versions] of Object.entries(rollouts)) {
+    for (const [version, rolloutPercentage] of Object.entries(versions)) {
+      const { count } = await db.release.updateMany({
+        where: { type, version },
+        data: { rolloutPercentage },
+      });
+      if (count !== 1)
+        throw new Error(`No release ${type} ${version} to set to ${rolloutPercentage}%`);
+    }
+  }
+}
+
+export async function getRelease(query: Record<string, string>) {
+  const response = await api(`/releases?${new URLSearchParams(query)}`);
+  return { status: response.status, body: await response.json() };
+}
+
+/**
+ * A cookie-session cookie pair, signed like the API signs it. Only the
+ * WebSocket client route accepts the id_token without asking Google, so the
+ * signaling tests double as the check that this signature is right.
+ */
+export function sessionCookie(session: object, secret = inject("cookieSecret")) {
+  const value = Buffer.from(JSON.stringify(session)).toString("base64");
+  const signature = createHmac("sha1", secret)
+    .update(`session=${value}`)
+    .digest("base64")
+    .replace(/\/|\+|=/g, c => ({ "/": "_", "+": "-", "=": "" })[c]!);
+  return `session=${value}; session.sig=${signature}`;
+}
+
+/** A JWT that decodes but is not signed by Google. */
+export function unsignedIdToken(claims: Record<string, unknown>) {
+  const part = (value: object) =>
+    Buffer.from(JSON.stringify(value)).toString("base64url");
+  const exp = Math.floor(Date.now() / 1000) + 3600;
+  return `${part({ alg: "RS256", kid: "e2e" })}.${part({ iss: "https://accounts.google.com", exp, ...claims })}.c2ln`;
+}
+
+export async function createUserWithDevice(
+  googleId: string,
+  deviceId: string,
+  secretToken: string,
+) {
+  await db.user.create({
+    data: { googleId, device: { create: { id: deviceId, secretToken } } },
+  });
+}
+
+function wsUrl(path: string) {
+  return `${apiUrl.replace(/^http/, "ws")}${path}`;
+}
+
+/** Resolves with the open socket, or rejects with the HTTP status (or "closed") the server answered with. */
+export function openSocket(
+  path: string,
+  headers: Record<string, string> = {},
+): Promise<WebSocket> {
+  return new Promise((resolve, reject) => {
+    const ws = new WebSocket(wsUrl(path), { headers });
+    // Listen before the handshake completes: the server may send right after it
+    messages(ws);
+    ws.once("open", () => resolve(ws));
+    ws.once("unexpected-response", (_req, res) =>
+      reject(new Error(`rejected: ${res.statusCode}`)),
+    );
+    ws.once("error", () => reject(new Error("rejected: closed")));
+  });
+}
+
+type Inbox = { next(timeoutMs?: number): Promise<any> };
+const inboxes = new WeakMap<WebSocket, Inbox>();
+
+/** Collects JSON (or text) messages so tests can await the next one. */
+export function messages(ws: WebSocket): Inbox {
+  const existing = inboxes.get(ws);
+  if (existing) return existing;
+  const queue: unknown[] = [];
+  const waiters: ((value: unknown) => void)[] = [];
+  ws.on("message", data => {
+    const text = data.toString();
+    let value: unknown = text;
+    try {
+      value = JSON.parse(text);
+    } catch {
+      // plain text such as "pong"
+    }
+    const waiter = waiters.shift();
+    if (waiter) waiter(value);
+    else queue.push(value);
+  });
+  const inbox: Inbox = {
+    next(timeoutMs = 5000): Promise<any> {
+      if (queue.length > 0) return Promise.resolve(queue.shift());
+      return new Promise((resolve, reject) => {
+        const waiter = (value: unknown) => {
+          clearTimeout(timer);
+          resolve(value);
+        };
+        const timer = setTimeout(() => {
+          waiters.splice(waiters.indexOf(waiter), 1);
+          reject(new Error("no message"));
+        }, timeoutMs);
+        waiters.push(waiter);
+      });
+    },
+  };
+  inboxes.set(ws, inbox);
+  return inbox;
+}
+
+export function closed(ws: WebSocket, timeoutMs = 5000): Promise<void> {
+  if (ws.readyState === WebSocket.CLOSED) return Promise.resolve();
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error("socket stayed open")), timeoutMs);
+    ws.once("close", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
diff --git a/e2e/http.test.ts b/e2e/http.test.ts
new file mode 100644
index 0000000..3c839cd
--- /dev/null
+++ b/e2e/http.test.ts
@@ -0,0 +1,136 @@
+import { describe, expect, inject, it } from "vitest";
+import { api, sessionCookie, unsignedIdToken } from "./helpers";
+
+const appOrigin = inject("appOrigin");
+
+describe("health", () => {
+  it("answers / and /healthz", async () => {
+    const root = await api("/");
+    expect(root.status).toBe(200);
+    expect(await root.text()).toBe("OK");
+
+    const health = await api("/healthz");
+    expect(health.status).toBe(200);
+    const body = await health.json();
+    expect(body.ready).toBe(true);
+    expect(Math.abs(Date.parse(body.time) - Date.now())).toBeLessThan(60_000);
+  });
+});
+
+describe("response headers", () => {
+  it("include the security headers and do not name the framework", async () => {
+    const response = await api("/healthz");
+    expect(response.headers.get("x-content-type-options")).toBe("nosniff");
+    expect(response.headers.get("x-frame-options")).toBe("SAMEORIGIN");
+    expect(response.headers.get("strict-transport-security")).toMatch(/max-age=\d+/);
+    expect(response.headers.get("content-security-policy")).toContain(
+      "default-src 'self'",
+    );
+    expect(response.headers.has("x-powered-by")).toBe(false);
+  });
+});
+
+describe("CORS", () => {
+  const preflight = (origin: string) =>
+    api("/devices", {
+      method: "OPTIONS",
+      headers: { origin, "access-control-request-method": "GET" },
+    });
+
+  it("lets the web app call the API with its cookies", async () => {
+    const response = await preflight(appOrigin);
+    expect(response.status).toBe(204);
+    expect(response.headers.get("access-control-allow-origin")).toBe(appOrigin);
+    expect(response.headers.get("access-control-allow-credentials")).toBe("true");
+  });
+
+  it.each(["https://evil.example", "https://app.e2e.test.evil.example", "null"])(
+    "does not let %s read responses",
+    async origin => {
+      const response = await preflight(origin);
+      expect(response.headers.has("access-control-allow-origin")).toBe(false);
+
+      const simple = await api("/healthz", { headers: { origin } });
+      expect(simple.headers.has("access-control-allow-origin")).toBe(false);
+    },
+  );
+});
+
+describe("routes that need a signed-in user", () => {
+  const routes: [string, string][] = [
+    ["GET", "/me"],
+    ["GET", "/devices"],
+    ["GET", "/devices/some-device"],
+    ["PUT", "/devices/some-device"],
+    ["POST", "/webrtc/session"],
+    ["POST", "/webrtc/ice_config"],
+    ["POST", "/webrtc/turn_activity"],
+  ];
+
+  const call = (method: string, path: string, headers: Record<string, string> = {}) =>
+    api(path, {
+      method,
+      headers: { "content-type": "application/json", ...headers },
+      body: method === "GET" ? undefined : "{}",
+    });
+
+  it.each(routes)("%s %s answers 401 without a session", async (method, path) => {
+    const response = await call(method, path);
+    expect(response.status).toBe(401);
+    expect((await response.json()).name).toBe("Unauthorized");
+  });
+
+  it.each(routes)(
+    "%s %s answers 401 to an ID token Google did not sign",
+    async (method, path) => {
+      const cookie = sessionCookie({
+        id_token: unsignedIdToken({ sub: "e2e-attacker", email: "a@e2e.test" }),
+      });
+      expect((await call(method, path, { cookie })).status).toBe(401);
+    },
+  );
+
+  it("ignores a session cookie signed with another secret", async () => {
+    const cookie = sessionCookie(
+      { id_token: unsignedIdToken({ sub: "e2e-attacker" }) },
+      "not-the-secret",
+    );
+    expect((await api("/devices", { headers: { cookie } })).status).toBe(401);
+  });
+});
+
+describe("errors", () => {
+  it("answer an unknown route with 404", async () => {
+    expect((await api("/does-not-exist")).status).toBe(404);
+  });
+
+  it("do not include a stack trace", async () => {
+    const response = await api("/releases");
+    expect(response.status).toBe(400);
+    expect(await response.json()).toEqual({
+      name: "BadRequestError",
+      message: "Device ID is required",
+    });
+  });
+});
+
+describe("sign-in and sign-out", () => {
+  it("the OIDC callback passes the query on to the cookie-carrying route, on this host", async () => {
+    const response = await api("/oidc/callback?code=abc&state=xyz");
+    expect(response.status).toBe(200);
+    expect(await response.text()).toContain(`URL='/oidc/callback_o?code=abc&state=xyz'`);
+  });
+
+  it("logout clears the session cookie", async () => {
+    const response = await api("/logout", {
+      method: "POST",
+      // As behind the TLS-terminating proxy, which the secure cookie needs
+      headers: { "x-forwarded-proto": "https", cookie: sessionCookie({ id_token: "x" }) },
+    });
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ message: "Logged out" });
+    const setCookie = response.headers.getSetCookie().join("\n");
+    expect(setCookie).toMatch(/session=;/);
+    expect(setCookie).toMatch(/expires=Thu, 01 Jan 1970/i);
+  });
+});
diff --git a/e2e/release-sync.test.ts b/e2e/release-sync.test.ts
new file mode 100644
index 0000000..51fc6db
--- /dev/null
+++ b/e2e/release-sync.test.ts
@@ -0,0 +1,204 @@
+import { afterEach, describe, expect, inject, it } from "vitest";
+import {
+  api,
+  cdn,
+  db,
+  removeArtifact,
+  sha256,
+  syncReleases,
+  uploadArtifact,
+} from "./helpers";
+
+async function registered() {
+  return db.release.findMany({
+    select: {
+      type: true,
+      version: true,
+      artifacts: { select: { url: true, compatibleSkus: true }, orderBy: { id: "asc" } },
+    },
+    orderBy: [{ type: "asc" }, { id: "asc" }],
+  });
+}
+
+describe("release registration", () => {
+  it("registers each stable version in the bucket with the SKUs it has files for", async () => {
+    expect(await registered()).toEqual([
+      {
+        type: "app",
+        version: "0.3.0",
+        artifacts: [{ url: cdn("app/0.3.0/jetkvm_app"), compatibleSkus: ["jetkvm-v2"] }],
+      },
+      {
+        type: "app",
+        version: "0.4.0",
+        artifacts: [
+          {
+            url: cdn("app/0.4.0/skus/jetkvm-v2/jetkvm_app"),
+            compatibleSkus: ["jetkvm-v2"],
+          },
+          {
+            url: cdn("app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app"),
+            compatibleSkus: ["jetkvm-v2-sdmmc"],
+          },
+        ],
+      },
+      {
+        type: "app",
+        version: "0.5.0",
+        artifacts: [
+          {
+            url: cdn("app/0.5.0/skus/jetkvm-v2/jetkvm_app"),
+            compatibleSkus: ["jetkvm-v2"],
+          },
+        ],
+      },
+      {
+        type: "mini",
+        version: "0.5.3",
+        artifacts: [
+          {
+            url: cdn("mini/0.5.3/skus/jetkvm-mini-ethernet/jetkvm-mini.bin"),
+            compatibleSkus: ["jetkvm-mini-ethernet"],
+          },
+          {
+            url: cdn("mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini.bin"),
+            compatibleSkus: ["jetkvm-mini-wireless"],
+          },
+        ],
+      },
+      {
+        type: "mini",
+        version: "0.5.4",
+        artifacts: [
+          {
+            url: cdn("mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini.bin"),
+            compatibleSkus: ["jetkvm-mini-ethernet"],
+          },
+        ],
+      },
+      {
+        type: "system",
+        version: "0.2.0",
+        artifacts: [
+          { url: cdn("system/0.2.0/system.tar"), compatibleSkus: ["jetkvm-v2"] },
+        ],
+      },
+      {
+        type: "system",
+        version: "0.3.0",
+        artifacts: [
+          {
+            url: cdn("system/0.3.0/skus/jetkvm-v2/system.tar"),
+            compatibleSkus: ["jetkvm-v2"],
+          },
+          {
+            url: cdn("system/0.3.0/skus/jetkvm-v2-sdmmc/system.tar"),
+            compatibleSkus: ["jetkvm-v2-sdmmc"],
+          },
+        ],
+      },
+    ]);
+  });
+
+  describe("of a new upload", () => {
+    const keys = [
+      "app/0.4.5/skus/jetkvm-v2/jetkvm_app",
+      "app/0.4.5/skus/jetkvm-v2-sdmmc/jetkvm_app",
+    ];
+
+    afterEach(async () => {
+      await db.release.deleteMany({ where: { type: "app", version: "0.4.5" } });
+      await Promise.all(keys.map(removeArtifact));
+    });
+
+    it("waits for the upload to settle unless the uploader says it is complete, then starts at 0%", async () => {
+      const bodies = keys.map(key => Buffer.from(`new upload ${key}`));
+      await Promise.all(keys.map((key, i) => uploadArtifact(key, bodies[i])));
+
+      // The objects were just written, so an unattended run leaves the version for later
+      const unattended = await syncReleases();
+      expect(unattended.status).toBe(200);
+      expect(await unattended.json()).toMatchObject({ created: 0 });
+      expect(await db.release.count({ where: { type: "app", version: "0.4.5" } })).toBe(
+        0,
+      );
+
+      const vouched = await syncReleases({ type: "app", version: "0.4.5" });
+      expect(await vouched.json()).toMatchObject({ created: 1 });
+
+      const release = await db.release.findUniqueOrThrow({
+        where: { version_type: { version: "0.4.5", type: "app" } },
+        include: { artifacts: { orderBy: { id: "asc" } } },
+      });
+      expect(release.rolloutPercentage).toBe(0);
+      expect(
+        release.artifacts.map(({ url, hash, compatibleSkus }) => ({
+          url,
+          hash,
+          compatibleSkus,
+        })),
+      ).toEqual([
+        { url: cdn(keys[0]), hash: sha256(bodies[0]), compatibleSkus: ["jetkvm-v2"] },
+        {
+          url: cdn(keys[1]),
+          hash: sha256(bodies[1]),
+          compatibleSkus: ["jetkvm-v2-sdmmc"],
+        },
+      ]);
+    });
+  });
+
+  it("never changes a release that is already registered", async () => {
+    await db.release.updateMany({
+      where: { type: "app", version: "0.4.0" },
+      data: { rolloutPercentage: 37 },
+    });
+    const before = await db.release.findFirstOrThrow({
+      where: { type: "app", version: "0.4.0" },
+      include: { artifacts: true },
+    });
+
+    const response = await syncReleases({ type: "app", version: "0.4.0" });
+    expect(response.status).toBe(200);
+    expect(await response.json()).toMatchObject({ created: 0 });
+
+    const after = await db.release.findFirstOrThrow({
+      where: { type: "app", version: "0.4.0" },
+      include: { artifacts: true },
+    });
+    expect(after).toEqual(before);
+  });
+});
+
+describe("POST /releases/sync", () => {
+  const token = inject("syncToken");
+
+  it.each([
+    ["no Authorization header", {}],
+    ["a wrong token", { authorization: "Bearer not-the-token" }],
+    [
+      "the token in another letter case",
+      { authorization: `Bearer ${token.toUpperCase()}` },
+    ],
+    ["the token under another scheme", { authorization: `Basic ${token}` }],
+  ])("rejects %s with 401", async (_name, headers) => {
+    const response = await api("/releases/sync", { method: "POST", headers });
+    expect(response.status).toBe(401);
+  });
+
+  it("accepts the token with the scheme name in any case", async () => {
+    const response = await api("/releases/sync", {
+      method: "POST",
+      headers: { authorization: `bearer ${token}` },
+    });
+    expect(response.status).toBe(200);
+  });
+
+  it.each([
+    ["a type without a version", { type: "app" }],
+    ["a version without a type", { version: "0.4.0" }],
+    ["an unknown type", { type: "firmware", version: "0.4.0" }],
+  ])("rejects %s with 400", async (_name, body) => {
+    expect((await syncReleases(body)).status).toBe(400);
+  });
+});
diff --git a/e2e/releases.test.ts b/e2e/releases.test.ts
new file mode 100644
index 0000000..c589786
--- /dev/null
+++ b/e2e/releases.test.ts
@@ -0,0 +1,320 @@
+import { afterEach, describe, expect, it } from "vitest";
+import { DEVICE_IN_BUCKET } from "./fixtures";
+import {
+  api,
+  cdn,
+  db,
+  expectServable,
+  getRelease,
+  objectBytes,
+  removeArtifact,
+  setRollouts,
+  syncReleases,
+  uploadArtifact,
+} from "./helpers";
+
+// Selection rules are covered by test/releases.test.ts against a mocked S3.
+// These tests cover what only a real bucket, a real sync and HTTP show: the
+// files a device is sent exist and match the hash it is told to check.
+
+const V2 = "jetkvm-v2";
+const SDMMC = "jetkvm-v2-sdmmc";
+const MINI_ETH = "jetkvm-mini-ethernet";
+const MINI_WIFI = "jetkvm-mini-wireless";
+
+/** The offered versions and URLs, after checking every URL serves bytes with the offered hash. */
+async function offered(query: Record<string, string>) {
+  const { status, body } = await getRelease(query);
+  expect(status, JSON.stringify(body)).toBe(200);
+  if (body.appUrl) await expectServable(body.appUrl, body.appHash);
+  if (body.systemUrl) await expectServable(body.systemUrl, body.systemHash);
+  return {
+    appVersion: body.appVersion,
+    appUrl: body.appUrl,
+    systemVersion: body.systemVersion,
+    systemUrl: body.systemUrl,
+  };
+}
+
+const device = (bucket: keyof typeof DEVICE_IN_BUCKET, sku = V2) => ({
+  deviceId: DEVICE_IN_BUCKET[bucket],
+  sku,
+});
+
+describe("an update check", () => {
+  it("gets, for each SKU, files that exist and match the hashes it must verify", async () => {
+    await setRollouts({
+      app: { "0.4.0": 100 },
+      system: { "0.3.0": 100 },
+      mini: { "0.5.3": 100, "0.5.4": 100 },
+    });
+
+    expect(await offered(device(9, V2))).toEqual({
+      appVersion: "0.4.0",
+      appUrl: cdn("app/0.4.0/skus/jetkvm-v2/jetkvm_app"),
+      systemVersion: "0.3.0",
+      systemUrl: cdn("system/0.3.0/skus/jetkvm-v2/system.tar"),
+    });
+    expect(await offered(device(9, SDMMC))).toEqual({
+      appVersion: "0.4.0",
+      appUrl: cdn("app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app"),
+      systemVersion: "0.3.0",
+      systemUrl: cdn("system/0.3.0/skus/jetkvm-v2-sdmmc/system.tar"),
+    });
+    expect(await offered(device(9, MINI_ETH))).toEqual({
+      appVersion: undefined,
+      appUrl: undefined,
+      systemVersion: "0.5.4",
+      systemUrl: cdn("mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini.bin"),
+    });
+    // 0.5.4 has no wireless build
+    expect(await offered(device(9, MINI_WIFI))).toMatchObject({
+      systemVersion: "0.5.3",
+      systemUrl: cdn("mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini.bin"),
+    });
+  });
+
+  it("gets a signature URL only where the bucket has a .sig file", async () => {
+    await setRollouts({ app: { "0.4.0": 100 }, system: { "0.3.0": 100 } });
+
+    const v2 = (await getRelease(device(9, V2))).body;
+    expect(v2.appSigUrl).toBe(cdn("app/0.4.0/skus/jetkvm-v2/jetkvm_app.sig"));
+    expect(
+      (await objectBytes("app/0.4.0/skus/jetkvm-v2/jetkvm_app.sig")).length,
+    ).toBeGreaterThan(0);
+    expect(v2).not.toHaveProperty("systemSigUrl");
+    expect((await getRelease(device(9, SDMMC))).body).not.toHaveProperty("appSigUrl");
+  });
+
+  it("puts devices into a staged rollout by a bucket that does not change between versions of the API", async () => {
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 50 }, system: { "0.3.0": 100 } });
+    expect((await offered(device(0))).appVersion).toBe("0.5.0");
+    expect((await offered(device(49))).appVersion).toBe("0.5.0");
+    expect((await offered(device(50))).appVersion).toBe("0.4.0");
+    expect((await offered(device(99))).appVersion).toBe("0.4.0");
+
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 81 }, system: { "0.3.0": 100 } });
+    expect((await offered(device(81))).appVersion).toBe("0.4.0");
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 82 }, system: { "0.3.0": 100 } });
+    expect((await offered(device(81))).appVersion).toBe("0.5.0");
+  });
+
+  it("never sends a device a file built for other hardware", async () => {
+    // 0.5.0 has no SDMMC build; 0.3.0 and 0.2.0 predate the SKU layout and are JetKVM v2 only
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 100 }, system: { "0.3.0": 100 } });
+    expect((await offered(device(9, SDMMC))).appUrl).toBe(
+      cdn("app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app"),
+    );
+
+    await setRollouts({ app: { "0.3.0": 100 }, system: { "0.2.0": 100 } });
+    expect(await offered(device(9, V2))).toMatchObject({
+      appUrl: cdn("app/0.3.0/jetkvm_app"),
+      systemUrl: cdn("system/0.2.0/system.tar"),
+    });
+    expect((await getRelease(device(9, SDMMC))).status).toBe(404);
+  });
+
+  it("on the prerelease channel gets the newest build in the bucket, whatever the rollout", async () => {
+    await setRollouts({});
+    expect(await offered({ ...device(99, V2), prerelease: "true" })).toEqual({
+      appVersion: "0.6.0-beta.1",
+      appUrl: cdn("app/0.6.0-beta.1/skus/jetkvm-v2/jetkvm_app"),
+      systemVersion: "0.4.0-rc.1",
+      systemUrl: cdn("system/0.4.0-rc.1/skus/jetkvm-v2/system.tar"),
+    });
+  });
+});
+
+describe("a release from upload to full rollout", () => {
+  const keys = [
+    "app/0.4.6/skus/jetkvm-v2/jetkvm_app",
+    "app/0.4.6/skus/jetkvm-v2-sdmmc/jetkvm_app",
+  ];
+
+  afterEach(async () => {
+    await db.release.deleteMany({ where: { type: "app", version: "0.4.6" } });
+    await Promise.all(keys.map(removeArtifact));
+  });
+
+  const setApp046 = (rolloutPercentage: number) =>
+    db.release.updateMany({
+      where: { type: "app", version: "0.4.6" },
+      data: { rolloutPercentage },
+    });
+
+  it("reaches no device at 0%, the devices in the bucket while staged, and every device at 100%", async () => {
+    // app 0.5.0 stays registered at 0%: a newer unreleased version must not get in the way
+    await setRollouts({ app: { "0.4.0": 100 }, system: { "0.3.0": 100 } });
+    await Promise.all(
+      keys.map(key => uploadArtifact(key, Buffer.from(`release ${key}`))),
+    );
+    expect(
+      await (await syncReleases({ type: "app", version: "0.4.6" })).json(),
+    ).toMatchObject({
+      created: 1,
+    });
+
+    // Registered at 0%: every device keeps 0.4.0
+    for (const bucket of [0, 9, 81, 99] as const) {
+      expect((await offered(device(bucket))).appVersion).toBe("0.4.0");
+    }
+
+    await setApp046(30);
+    expect(await offered(device(9))).toMatchObject({
+      appVersion: "0.4.6",
+      appUrl: cdn(keys[0]),
+    });
+    expect(await offered(device(9, SDMMC))).toMatchObject({
+      appVersion: "0.4.6",
+      appUrl: cdn(keys[1]),
+    });
+    expect((await offered(device(81))).appVersion).toBe("0.4.0");
+    expect((await offered(device(81, SDMMC))).appVersion).toBe("0.4.0");
+
+    await setApp046(100);
+    for (const bucket of [0, 81, 99] as const) {
+      expect((await offered(device(bucket))).appVersion).toBe("0.4.6");
+      expect((await offered(device(bucket, SDMMC))).appUrl).toBe(cdn(keys[1]));
+    }
+  });
+});
+
+describe("the first staged release that has a build for a new SKU", () => {
+  // app 0.3.0 predates the SKU layout (JetKVM v2 only); 0.4.0 is the first with an SDMMC build
+  it("reaches the new SKU's devices in the bucket, and the rest get 404 until it is rolled out", async () => {
+    await setRollouts({ app: { "0.3.0": 100, "0.4.0": 30 }, system: { "0.3.0": 100 } });
+
+    expect(await offered(device(9, SDMMC))).toMatchObject({
+      appVersion: "0.4.0",
+      appUrl: cdn("app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app"),
+    });
+    const outside = await getRelease(device(81, SDMMC));
+    expect(outside.status).toBe(404);
+    expect(outside.body.name).toBe("NotFoundError");
+
+    // The original JetKVM is not affected
+    expect((await offered(device(9, V2))).appVersion).toBe("0.4.0");
+    expect((await offered(device(81, V2))).appUrl).toBe(cdn("app/0.3.0/jetkvm_app"));
+  });
+});
+
+describe("an update check that pins one part", () => {
+  it("gets the other part as a background check would: rolled out, with a build for the SKU", async () => {
+    // system 0.3.0 is registered but not released; app 0.5.0 has no SDMMC build
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 100 }, system: { "0.2.0": 100 } });
+
+    expect(await offered({ ...device(9, V2), appVersion: "0.5.0" })).toMatchObject({
+      appVersion: "0.5.0",
+      systemVersion: "0.2.0",
+    });
+    expect(await offered({ ...device(9, V2), systemVersion: "0.3.0" })).toMatchObject({
+      appVersion: "0.5.0",
+      systemVersion: "0.3.0",
+    });
+
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 100 }, system: { "0.3.0": 100 } });
+    expect(await offered({ ...device(9, SDMMC), systemVersion: "0.3.0" })).toMatchObject({
+      appVersion: "0.4.0",
+      appUrl: cdn("app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app"),
+      systemVersion: "0.3.0",
+    });
+  });
+
+  it("follows the device's rollout bucket for the other part", async () => {
+    await setRollouts({ app: { "0.4.0": 100, "0.5.0": 50 }, system: { "0.3.0": 100 } });
+    expect((await offered({ ...device(9), systemVersion: "0.3.0" })).appVersion).toBe(
+      "0.5.0",
+    );
+    expect((await offered({ ...device(81), systemVersion: "0.3.0" })).appVersion).toBe(
+      "0.4.0",
+    );
+  });
+});
+
+describe("a SKU missing from the newest upload", () => {
+  // app 0.5.0 has no SDMMC build; mini 0.5.4 has no wireless build
+
+  it("does not hide the newest version that has it on the prerelease channel", async () => {
+    await setRollouts({});
+    expect(await offered({ ...device(9, MINI_WIFI), prerelease: "true" })).toMatchObject({
+      systemVersion: "0.5.3",
+      systemUrl: cdn("mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini.bin"),
+    });
+  });
+
+  it.each([
+    "/releases/app/latest?sku=jetkvm-v2-sdmmc",
+    "/releases/system_recovery/latest?sku=jetkvm-mini-wireless",
+  ])("makes %s answer 404, not an older file", async path => {
+    expect((await api(path)).status).toBe(404);
+  });
+});
+
+describe("an update check that cannot be answered", () => {
+  it("gets 404 while every release is registered at 0%", async () => {
+    await setRollouts({});
+    for (const sku of [V2, SDMMC, MINI_ETH]) {
+      expect((await getRelease(device(9, sku))).status).toBe(404);
+    }
+  });
+
+  it("gets 404 for a pinned version that does not exist", async () => {
+    await setRollouts({ app: { "0.4.0": 100 }, system: { "0.3.0": 100 } });
+    const { status } = await getRelease({
+      ...device(9),
+      appVersion: "9.9.9",
+      systemVersion: "0.3.0",
+    });
+    expect(status).toBe(404);
+  });
+
+  it.each([
+    ["no device ID", ""],
+    ["an empty device ID", "deviceId="],
+    ["an unknown SKU", `deviceId=${DEVICE_IN_BUCKET[9]}&sku=jetkvm-v3`],
+  ])("with %s gets 400 and no release", async (_name, query) => {
+    await setRollouts({ app: { "0.4.0": 100 }, system: { "0.3.0": 100 } });
+    const response = await api(`/releases?${query}`);
+    expect(response.status).toBe(400);
+    const body = await response.json();
+    expect(body.name).toBe("BadRequestError");
+    expect(body).not.toHaveProperty("appUrl");
+    expect(body).not.toHaveProperty("stack");
+  });
+});
+
+describe("the latest-artifact redirects used by build tooling", () => {
+  it.each([
+    ["/releases/app/latest", "app/0.5.0/skus/jetkvm-v2/jetkvm_app"],
+    [
+      "/releases/app/latest?prerelease=true",
+      "app/0.6.0-beta.1/skus/jetkvm-v2/jetkvm_app",
+    ],
+    ["/releases/system_recovery/latest", "system/0.3.0/skus/jetkvm-v2/update.img"],
+    [
+      "/releases/system_recovery/latest?sku=jetkvm-v2-sdmmc",
+      "system/0.3.0/skus/jetkvm-v2-sdmmc/update_sd.img.zip",
+    ],
+    [
+      "/releases/system_recovery/latest?prerelease=true",
+      "system/0.4.0-rc.1/skus/jetkvm-v2/update.img",
+    ],
+    [
+      "/releases/system_recovery/latest?sku=jetkvm-mini-ethernet",
+      "mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini-full.bin",
+    ],
+  ])("%s redirects to %s, which exists", async (path, key) => {
+    const response = await api(path);
+    expect(response.status).toBe(302);
+    expect(response.headers.get("location")).toBe(cdn(key));
+    expect((await objectBytes(key)).length).toBeGreaterThan(0);
+  });
+
+  it.each([
+    "/releases/app/latest?sku=jetkvm-mini-ethernet",
+    "/releases/app/latest?sku=jetkvm-v3",
+    "/releases/system_recovery/latest?sku=jetkvm-v3",
+  ])("%s is a bad request", async path => {
+    expect((await api(path)).status).toBe(400);
+  });
+});
diff --git a/e2e/signaling.test.ts b/e2e/signaling.test.ts
new file mode 100644
index 0000000..f79e10b
--- /dev/null
+++ b/e2e/signaling.test.ts
@@ -0,0 +1,229 @@
+import { afterEach, beforeEach, describe, expect, inject, it } from "vitest";
+import { connect } from "node:net";
+import type WebSocket from "ws";
+import {
+  api,
+  closed,
+  createUserWithDevice,
+  db,
+  messages,
+  openSocket,
+  sessionCookie,
+  unsignedIdToken,
+} from "./helpers";
+
+const OWNER = "e2e-sig-owner";
+const OTHER_USER = "e2e-sig-other";
+const DEVICE = "e2e-sig-device";
+const SECRET = "e2e-sig-secret";
+const DEVICE_IP = "203.0.113.7";
+
+const sockets: WebSocket[] = [];
+async function track(socket: Promise<WebSocket>) {
+  const ws = await socket;
+  sockets.push(ws);
+  return ws;
+}
+
+function connectDevice(headers: Record<string, string> = {}) {
+  return track(
+    openSocket("/", {
+      authorization: `Bearer ${SECRET}`,
+      "x-device-id": DEVICE,
+      "x-app-version": "0.5.0",
+      "x-device-sku": "jetkvm-v2-sdmmc",
+      "x-real-ip": DEVICE_IP,
+      ...headers,
+    }),
+  );
+}
+
+const idTokenFor = (sub: string) => unsignedIdToken({ sub });
+
+function connectClient(sub = OWNER, deviceId = DEVICE) {
+  return track(
+    openSocket(`/webrtc/signaling/client?id=${deviceId}`, {
+      cookie: sessionCookie({ id_token: idTokenFor(sub) }),
+    }),
+  );
+}
+
+beforeEach(async () => {
+  await db.device.deleteMany({ where: { id: DEVICE } });
+  await db.user.deleteMany({ where: { googleId: { in: [OWNER, OTHER_USER] } } });
+  await createUserWithDevice(OWNER, DEVICE, SECRET);
+  await db.user.create({ data: { googleId: OTHER_USER } });
+});
+
+afterEach(async () => {
+  await Promise.all(
+    sockets.splice(0).map(ws => {
+      ws.close();
+      return closed(ws).catch(() => undefined);
+    }),
+  );
+});
+
+describe("device connection", () => {
+  it.each([
+    ["no token", { authorization: "" }],
+    ["a wrong token", { authorization: "Bearer wrong" }],
+    ["another device's ID", { "x-device-id": "e2e-sig-someone-else" }],
+    ["no device ID", { "x-device-id": "" }],
+  ])("is refused with %s", async (_name, headers) => {
+    await expect(connectDevice(headers)).rejects.toThrow(/rejected/);
+  });
+
+  it("records when the device was last seen after it disconnects", async () => {
+    const device = await connectDevice();
+    device.close();
+    await closed(device);
+
+    await expect
+      .poll(
+        async () =>
+          (await db.device.findUniqueOrThrow({ where: { id: DEVICE } })).lastSeen,
+      )
+      .not.toBeNull();
+  });
+
+  it("replaces the earlier connection when the same device connects again", async () => {
+    const first = await connectDevice();
+    const firstClosed = closed(first);
+    await connectDevice({ "x-app-version": "0.5.1" });
+    await firstClosed;
+
+    const client = messages(await connectClient());
+    expect(await client.next()).toEqual({
+      type: "device-metadata",
+      data: { deviceVersion: "0.5.1", deviceSku: "jetkvm-v2-sdmmc" },
+    });
+  });
+});
+
+describe("client connection", () => {
+  it("is refused without a session", async () => {
+    await expect(
+      track(openSocket(`/webrtc/signaling/client?id=${DEVICE}`)),
+    ).rejects.toThrow(/rejected/);
+  });
+
+  it("is refused with a session cookie signed with another secret", async () => {
+    await connectDevice();
+    const cookie = sessionCookie({ id_token: idTokenFor(OWNER) }, "not-the-secret");
+    await expect(
+      track(openSocket(`/webrtc/signaling/client?id=${DEVICE}`, { cookie })),
+    ).rejects.toThrow(/rejected/);
+  });
+
+  it("is refused for a device that belongs to another user", async () => {
+    await connectDevice();
+    await expect(connectClient(OTHER_USER)).rejects.toThrow(/rejected/);
+  });
+
+  it("answers 404 while the device is offline", async () => {
+    await expect(connectClient()).rejects.toThrow("rejected: 404");
+  });
+
+  it("survives an upgrade request whose target is not a valid URL", async () => {
+    const { hostname, port } = new URL(inject("apiUrl"));
+    await new Promise<void>(resolve => {
+      const socket = connect(Number(port), hostname, () =>
+        socket.write(
+          "GET http://[ HTTP/1.1\r\nHost: e2e\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n" +
+            "Sec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n",
+        ),
+      );
+      socket.on("close", () => resolve());
+      socket.on("error", () => resolve());
+    });
+    expect((await api("/healthz")).status).toBe(200);
+  });
+
+  it("is refused on an unknown path", async () => {
+    await expect(track(openSocket("/webrtc/signaling/other"))).rejects.toThrow(
+      /rejected/,
+    );
+  });
+});
+
+describe("signaling between the owner and the device", () => {
+  it("tells the client which firmware and hardware the device runs", async () => {
+    await connectDevice();
+    const client = messages(await connectClient());
+    expect(await client.next()).toEqual({
+      type: "device-metadata",
+      data: { deviceVersion: "0.5.0", deviceSku: "jetkvm-v2-sdmmc" },
+    });
+  });
+
+  it("reports the original JetKVM for a device that sends an unknown SKU", async () => {
+    await connectDevice({ "x-device-sku": "jetkvm-v9" });
+    const client = messages(await connectClient());
+    expect((await client.next()).data.deviceSku).toBe("jetkvm-v2");
+  });
+
+  it("forwards the offer with the device's public IP, ICE servers and the user's ID token", async () => {
+    const device = messages(await connectDevice());
+    const clientWs = await connectClient();
+    const client = messages(clientWs);
+    await client.next(); // device-metadata
+
+    clientWs.send(JSON.stringify({ type: "offer", data: { sd: "offer-sdp" } }));
+    expect(await device.next()).toEqual({
+      type: "offer",
+      data: {
+        sd: "offer-sdp",
+        ip: DEVICE_IP,
+        iceServers: [inject("iceServer")],
+        OidcGoogle: idTokenFor(OWNER),
+      },
+    });
+  });
+
+  it("relays the answer and ICE candidates both ways", async () => {
+    const deviceWs = await connectDevice();
+    const device = messages(deviceWs);
+    const clientWs = await connectClient();
+    const client = messages(clientWs);
+    await client.next(); // device-metadata
+
+    deviceWs.send(JSON.stringify({ type: "answer", data: "answer-sdp" }));
+    expect(await client.next()).toEqual({ type: "answer", data: "answer-sdp" });
+
+    deviceWs.send(
+      JSON.stringify({ type: "new-ice-candidate", data: { candidate: "from-device" } }),
+    );
+    expect(await client.next()).toEqual({
+      type: "new-ice-candidate",
+      data: { candidate: "from-device" },
+    });
+
+    clientWs.send(
+      JSON.stringify({ type: "new-ice-candidate", data: { candidate: "from-client" } }),
+    );
+    expect(await device.next()).toEqual({
+      type: "new-ice-candidate",
+      data: { candidate: "from-client" },
+    });
+  });
+
+  it("answers the client's keepalive", async () => {
+    await connectDevice();
+    const clientWs = await connectClient();
+    const client = messages(clientWs);
+    await client.next(); // device-metadata
+
+    clientWs.send("ping");
+    expect(await client.next()).toBe("pong");
+  });
+
+  it("closes the client when the device goes away", async () => {
+    const deviceWs = await connectDevice();
+    const clientWs = await connectClient();
+    const clientClosed = closed(clientWs);
+
+    deviceWs.close();
+    await clientClosed;
+  });
+});
diff --git a/e2e/support.ts b/e2e/support.ts
new file mode 100644
index 0000000..f23a972
--- /dev/null
+++ b/e2e/support.ts
@@ -0,0 +1,44 @@
+import { createHash } from "node:crypto";
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+// Shared by global setup and the tests. Nothing here may call inject(), which
+// only works inside the test runner.
+
+export const sha256 = (bytes: Buffer | string) =>
+  createHash("sha256").update(bytes).digest("hex");
+
+export function s3Client(endpoint: string) {
+  return new S3Client({
+    endpoint,
+    region: "auto",
+    credentials: { accessKeyId: "e2e", secretAccessKey: "e2e" },
+  });
+}
+
+/** Uploads an artifact and its .sha256 sidecar, as the release script does. */
+export async function putArtifact(
+  s3: S3Client,
+  bucket: string,
+  key: string,
+  body: Buffer,
+) {
+  await Promise.all([
+    s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })),
+    s3.send(
+      new PutObjectCommand({
+        Bucket: bucket,
+        Key: `${key}.sha256`,
+        Body: `${sha256(body)}\n`,
+      }),
+    ),
+  ]);
+}
+
+/** POST /releases/sync as the upload script calls it. */
+export function postSync(apiUrl: string, token: string, body?: object) {
+  return fetch(`${apiUrl}/releases/sync`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
+    body: body ? JSON.stringify(body) : undefined,
+  });
+}
diff --git a/package.json b/package.json
index 2ddec3a..655c39f 100644
--- a/package.json
+++ b/package.json
@@ -16,7 +16,9 @@
     "build": "tsc",
     "test": "vitest run",
     "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage"
+    "test:coverage": "vitest run --coverage",
+    "test:e2e": "npm run build && vitest run --config vitest.e2e.config.ts",
+    "compare": "node scripts/compare-api.mts"
   },
   "engines": {
     "node": ">=22.23.4 <23"
diff --git a/scripts/compare-api.mts b/scripts/compare-api.mts
new file mode 100644
index 0000000..bce9df4
--- /dev/null
+++ b/scripts/compare-api.mts
@@ -0,0 +1,538 @@
+/**
+ * Sends the same requests to two API deployments and reports where the
+ * answers differ, grouped by cause.
+ *
+ *   npm run compare -- --candidate <url|local|staging> [options]
+ *
+ * Only safe requests are sent: GETs, CORS preflights, and POSTs that the
+ * API must reject before any handler runs.
+ */
+import { createHash } from "node:crypto";
+import { readFileSync, writeFileSync } from "node:fs";
+import { createInterface } from "node:readline/promises";
+import { parseArgs } from "node:util";
+
+const ALIASES: Record<string, string> = {
+  local: "http://localhost:3000",
+  staging: "https://staging-api.jetkvm.com",
+  production: "https://api.jetkvm.com",
+};
+
+const SKUS = [
+  "jetkvm-v2",
+  "jetkvm-v2-sdmmc",
+  "jetkvm-mini-ethernet",
+  "jetkvm-mini-wireless",
+];
+const MINI_SKUS = new Set(["jetkvm-mini-ethernet", "jetkvm-mini-wireless"]);
+const AREAS = ["releases", "redirects", "auth", "http", "replay"] as const;
+type Area = (typeof AREAS)[number];
+
+const COMPARED_HEADERS = [
+  "content-type",
+  "location",
+  "access-control-allow-origin",
+  "access-control-allow-credentials",
+  "x-content-type-options",
+  "x-frame-options",
+  "strict-transport-security",
+  "x-powered-by",
+];
+
+const USAGE = `Usage: npm run compare -- --candidate <url|local|staging> [options]
+
+  --candidate <url>     API under test (local = ${ALIASES.local}, staging = ${ALIASES.staging})
+  --reference <url>     API to compare against (default ${ALIASES.production})
+  --only <areas>        Comma-separated: ${AREAS.join(", ")} (default: all but replay)
+  --sku <sku>           Only this SKU; repeat for more (default: ${SKUS.join(", ")})
+  --device <id>         Add a device ID; repeat for more (default: one per 5% rollout bucket)
+  --requests <file>     Also replay these GET paths, one per line (# starts a comment)
+  --json <file>         Write every request and its result to this file
+  --concurrency <n>     Requests in flight per API (default 8)
+  --examples <n>        Examples shown per group (default 3)
+`;
+
+interface Probe {
+  area: Area;
+  /** Requests of one kind share a label; differences are grouped by it. */
+  label: string;
+  method: string;
+  path: string;
+  headers?: Record<string, string>;
+  /** Pinned requests do not depend on the rollout table. */
+  rolloutDependent?: boolean;
+}
+
+interface Answer {
+  status: number | string;
+  headers: Record<string, string>;
+  body: unknown;
+}
+
+interface Difference {
+  field: string;
+  reference: unknown;
+  candidate: unknown;
+}
+
+// ---------------------------------------------------------------- arguments
+
+const { values: args } = parseArgs({
+  options: {
+    candidate: { type: "string" },
+    reference: { type: "string", default: ALIASES.production },
+    only: { type: "string" },
+    sku: { type: "string", multiple: true },
+    device: { type: "string", multiple: true },
+    requests: { type: "string" },
+    json: { type: "string" },
+    concurrency: { type: "string", default: "8" },
+    examples: { type: "string", default: "3" },
+    help: { type: "boolean", short: "h" },
+  },
+});
+
+if (args.help || !args.candidate) {
+  console.log(USAGE);
+  process.exit(args.help ? 0 : 2);
+}
+
+const resolveTarget = (value: string) => (ALIASES[value] ?? value).replace(/\/+$/, "");
+const candidate = resolveTarget(args.candidate);
+const reference = resolveTarget(args.reference!);
+
+const areas = new Set<Area>(
+  args.only
+    ? args.only.split(",").map(area => {
+        if (!AREAS.includes(area as Area))
+          fail(`Unknown area "${area}". Areas: ${AREAS.join(", ")}`);
+        return area as Area;
+      })
+    : AREAS.filter(area => area !== "replay"),
+);
+if (args.requests) areas.add("replay");
+
+const skus = args.sku?.length ? args.sku : SKUS;
+const concurrency = Number(args.concurrency);
+const exampleCount = Number(args.examples);
+
+function fail(message: string): never {
+  console.error(message);
+  process.exit(2);
+}
+
+// ---------------------------------------------------------------- HTTP
+
+async function send(base: string, probe: Probe): Promise<Answer> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      const response = await fetch(base + probe.path, {
+        method: probe.method,
+        headers: probe.headers,
+        redirect: "manual",
+        signal: AbortSignal.timeout(20_000),
+      });
+      const text = await response.text();
+      if (response.status >= 500 && attempt < 2)
+        throw new Error(`HTTP ${response.status}`);
+
+      const headers: Record<string, string> = {};
+      for (const name of COMPARED_HEADERS) {
+        const value = response.headers.get(name);
+        if (value !== null) headers[name] = value;
+      }
+      let body: unknown = text;
+      try {
+        body = JSON.parse(text);
+      } catch {
+        // not JSON
+      }
+      return { status: response.status, headers, body };
+    } catch (error) {
+      if (attempt >= 2)
+        return { status: `error: ${(error as Error).message}`, headers: {}, body: null };
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+  }
+}
+
+async function pool<T, R>(
+  items: T[],
+  limit: number,
+  work: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let next = 0;
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, async () => {
+      while (next < items.length) {
+        const index = next++;
+        results[index] = await work(items[index]);
+      }
+    }),
+  );
+  return results;
+}
+
+// ---------------------------------------------------------------- requests
+
+/** One device ID per 5% rollout bucket, plus bucket 99, using the API's bucketing (md5 % 100). */
+function devicesAcrossBuckets(): string[] {
+  const bucket = (id: string) =>
+    parseInt(createHash("md5").update(id).digest("hex").slice(0, 8), 16) % 100;
+  const wanted = new Set([...Array.from({ length: 20 }, (_, i) => i * 5), 99]);
+  const ids: string[] = [];
+  for (let n = 0; wanted.size > 0 && n < 100_000; n++) {
+    const id = `compare-device-${n}`;
+    if (wanted.delete(bucket(id))) ids.push(id);
+  }
+  return ids;
+}
+
+const query = (params: Record<string, string | undefined>) =>
+  new URLSearchParams(
+    Object.entries(params).filter(([, v]) => v !== undefined) as [string, string][],
+  ).toString();
+
+/**
+ * Every registered version of one kind, newest first, as the reference API
+ * reports it: pinned requests ignore the rollout, so "<x" walks down the list.
+ */
+async function discoverVersions(sku: string, kind: "app" | "system"): Promise<string[]> {
+  const versions: string[] = [];
+  // ">=0.0.0" would normalize to "*", which the API treats as not pinned
+  let range = "<999999.0.0";
+  for (let i = 0; i < 200; i++) {
+    const answer = await send(reference, {
+      area: "releases",
+      label: "discovery",
+      method: "GET",
+      path: `/releases?${query({ deviceId: "compare-discovery", sku, [`${kind}Version`]: range })}`,
+    });
+    const version = (answer.body as Record<string, string> | null)?.[`${kind}Version`];
+    if (answer.status !== 200 || !version || versions.includes(version)) break;
+    versions.push(version);
+    range = `<${version}`;
+  }
+  return versions;
+}
+
+async function releaseProbes(devices: string[]): Promise<Probe[]> {
+  const probes: Probe[] = [];
+  const add = (
+    label: string,
+    params: Record<string, string | undefined>,
+    rolloutDependent = false,
+  ) =>
+    probes.push({
+      area: "releases",
+      label,
+      method: "GET",
+      path: `/releases?${query(params)}`,
+      rolloutDependent,
+    });
+
+  // One request at a time per chain, the chains side by side
+  const chains = skus.flatMap(sku =>
+    (MINI_SKUS.has(sku) ? (["system"] as const) : (["app", "system"] as const)).map(
+      kind => ({ sku, kind }),
+    ),
+  );
+  const discovered = await Promise.all(
+    chains.map(async chain => ({
+      ...chain,
+      versions: await discoverVersions(chain.sku, chain.kind),
+    })),
+  );
+
+  // A chain stops at a version with no build for its SKU, so every SKU of a
+  // release type is probed with the versions any of its SKUs found.
+  const releaseType = (sku: string, kind: string) => (MINI_SKUS.has(sku) ? "mini" : kind);
+  const versionsByType = new Map<string, Set<string>>();
+  for (const { sku, kind, versions } of discovered) {
+    const type = releaseType(sku, kind);
+    versionsByType.set(type, new Set([...(versionsByType.get(type) ?? []), ...versions]));
+  }
+
+  for (const sku of skus) {
+    for (const deviceId of devices) add(`stable ${sku}`, { deviceId, sku }, true);
+    add(`prerelease ${sku}`, { deviceId: devices[0], sku, prerelease: "true" });
+
+    for (const { kind } of chains.filter(chain => chain.sku === sku)) {
+      for (const version of versionsByType.get(releaseType(sku, kind)) ?? []) {
+        add(`pinned ${kind} ${sku}`, {
+          deviceId: devices[0],
+          sku,
+          [`${kind}Version`]: version,
+        });
+        add(`pinned ${kind} range ${sku}`, {
+          deviceId: devices[0],
+          sku,
+          [`${kind}Version`]: `<${version}`,
+        });
+      }
+    }
+  }
+
+  add("no SKU", { deviceId: devices[0] }, true);
+  add("invalid request", {});
+  add("invalid request", { deviceId: "" });
+  add("invalid request", { deviceId: devices[0], sku: "jetkvm-unknown" });
+  add("invalid request", {
+    deviceId: devices[0],
+    appVersion: "9999.0.0",
+    systemVersion: "9999.0.0",
+  });
+  return probes;
+}
+
+function redirectProbes(): Probe[] {
+  const probes: Probe[] = [];
+  for (const path of ["/releases/app/latest", "/releases/system_recovery/latest"]) {
+    for (const sku of skus) {
+      for (const prerelease of [undefined, "true"]) {
+        probes.push({
+          area: "redirects",
+          label: path,
+          method: "GET",
+          path: `${path}?${query({ sku, prerelease })}`,
+        });
+      }
+    }
+  }
+  return probes;
+}
+
+function authProbes(): Probe[] {
+  const routes: [string, string][] = [
+    ["GET", "/me"],
+    ["GET", "/devices"],
+    ["GET", "/devices/compare-device"],
+    ["PUT", "/devices/compare-device"],
+    ["POST", "/webrtc/session"],
+    ["POST", "/webrtc/ice_config"],
+    ["POST", "/webrtc/turn_activity"],
+    ["POST", "/releases/sync"],
+  ];
+  return routes.map(([method, path]) => ({
+    area: "auth",
+    label: `${method} ${path}`,
+    method,
+    path,
+  }));
+}
+
+function httpProbes(): Probe[] {
+  return [
+    { area: "http", label: "root", method: "GET", path: "/" },
+    { area: "http", label: "health", method: "GET", path: "/healthz" },
+    {
+      area: "http",
+      label: "unknown route",
+      method: "GET",
+      path: "/compare-does-not-exist",
+    },
+    {
+      area: "http",
+      label: "CORS from another origin",
+      method: "OPTIONS",
+      path: "/devices",
+      headers: {
+        origin: "https://compare.invalid",
+        "access-control-request-method": "GET",
+      },
+    },
+  ];
+}
+
+function replayProbes(file: string): Probe[] {
+  return readFileSync(file, "utf8")
+    .split("\n")
+    .map(line => line.trim())
+    .filter(line => line && !line.startsWith("#"))
+    .map(path => ({
+      area: "replay" as const,
+      label: path.split("?")[0],
+      method: "GET",
+      path,
+      rolloutDependent: true,
+    }));
+}
+
+// ---------------------------------------------------------------- comparison
+
+/** Values that differ on every call. */
+function normalize(body: unknown): unknown {
+  if (body && typeof body === "object" && !Array.isArray(body)) {
+    const { time: _time, ...rest } = body as Record<string, unknown>;
+    return rest;
+  }
+  return body;
+}
+
+function differences(ref: Answer, cand: Answer): Difference[] {
+  const found: Difference[] = [];
+  const walk = (field: string, a: unknown, b: unknown) => {
+    if (
+      a &&
+      b &&
+      typeof a === "object" &&
+      typeof b === "object" &&
+      !Array.isArray(a) &&
+      !Array.isArray(b)
+    ) {
+      const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
+      for (const key of [...keys].sort()) {
+        walk(field ? `${field}.${key}` : key, (a as any)[key], (b as any)[key]);
+      }
+    } else if (JSON.stringify(a) !== JSON.stringify(b)) {
+      found.push({ field, reference: a, candidate: b });
+    }
+  };
+  walk("status", ref.status, cand.status);
+  walk("header", ref.headers, cand.headers);
+  walk("body", normalize(ref.body), normalize(cand.body));
+  return found;
+}
+
+// ---------------------------------------------------------------- report
+
+const show = (value: unknown) =>
+  value === undefined ? "(absent)" : JSON.stringify(value);
+
+async function main() {
+  console.log(`Reference  ${reference}`);
+  console.log(`Candidate  ${candidate}\n`);
+
+  const healthProbe: Probe = {
+    area: "http",
+    label: "health",
+    method: "GET",
+    path: "/healthz",
+  };
+  const health = await Promise.all(
+    [reference, candidate].map(base => send(base, healthProbe)),
+  );
+  for (const [i, base] of [reference, candidate].entries()) {
+    if (health[i].status !== 200)
+      fail(`${base}/healthz answered ${health[i].status}; stopping.`);
+  }
+
+  const devices = [...devicesAcrossBuckets(), ...(args.device ?? [])];
+  const probes: Probe[] = [];
+  if (areas.has("releases")) probes.push(...(await releaseProbes(devices)));
+  if (areas.has("redirects")) probes.push(...redirectProbes());
+  if (areas.has("auth")) probes.push(...authProbes());
+  if (areas.has("http")) probes.push(...httpProbes());
+  if (areas.has("replay") && args.requests) probes.push(...replayProbes(args.requests));
+
+  let done = 0;
+  const results = await pool(probes, concurrency, async probe => {
+    const [ref, cand] = await Promise.all([
+      send(reference, probe),
+      send(candidate, probe),
+    ]);
+    if (++done % 25 === 0 || done === probes.length) {
+      process.stderr.write(`\r${done}/${probes.length} requests`);
+    }
+    return {
+      probe,
+      reference: ref,
+      candidate: cand,
+      differences: differences(ref, cand),
+    };
+  });
+  process.stderr.write("\n\n");
+
+  // Group by area, label and field, so one cause shows once however many requests it touches
+  const groups = new Map<
+    string,
+    { area: Area; label: string; field: string; members: typeof results }
+  >();
+  for (const result of results) {
+    for (const { field } of result.differences) {
+      const { area, label } = result.probe;
+      const key = JSON.stringify([area, label, field]);
+      if (!groups.has(key)) groups.set(key, { area, label, field, members: [] });
+      groups.get(key)!.members.push(result);
+    }
+  }
+
+  // In a terminal, each group of differences must be acknowledged as expected by the person
+  // running the compare. Nothing is remembered between runs.
+  const prompt = process.stdin.isTTY
+    ? createInterface({ input: process.stdin, output: process.stdout })
+    : undefined;
+  const sorted = [...groups.values()].sort((a, b) => b.members.length - a.members.length);
+  const acknowledged: { area: Area; label: string; field: string; expected: boolean }[] =
+    [];
+  for (const [index, { area, label, field, members }] of sorted.entries()) {
+    console.log(
+      `✗ [${index + 1}/${sorted.length}] ${area}: ${label}: ${field}  (${members.length} request${members.length === 1 ? "" : "s"})`,
+    );
+    for (const member of members.slice(0, exampleCount)) {
+      const difference = member.differences.find(d => d.field === field)!;
+      console.log(`    ${member.probe.method} ${member.probe.path}`);
+      console.log(`      reference ${show(difference.reference)}`);
+      console.log(`      candidate ${show(difference.candidate)}`);
+    }
+    if (prompt) {
+      // End of input (Ctrl+D) counts as "no"
+      const answer = await prompt
+        .question("  Is this difference expected? [y/N] ")
+        .catch(() => "");
+      acknowledged.push({
+        area,
+        label,
+        field,
+        expected: /^y(es)?$/i.test(answer.trim()),
+      });
+    }
+    console.log();
+  }
+  prompt?.close();
+
+  const differing = results.filter(result => result.differences.length > 0);
+  for (const area of areas) {
+    const inArea = results.filter(result => result.probe.area === area);
+    if (inArea.length === 0) continue;
+    const bad = inArea.filter(result => result.differences.length > 0).length;
+    console.log(
+      `${bad === 0 ? "✓" : "✗"} ${area.padEnd(10)} ${inArea.length - bad}/${inArea.length} identical`,
+    );
+  }
+
+  if (differing.length > 0 && differing.every(result => result.probe.rolloutDependent)) {
+    console.log(
+      "\nEvery difference is in a request that depends on the rollout table, and every pinned request matches.\n" +
+        "The rollout percentages in the two databases probably differ.",
+    );
+  }
+
+  if (args.json) {
+    writeFileSync(
+      args.json,
+      JSON.stringify({ reference, candidate, acknowledged, results }, null, 2),
+    );
+    console.log(`\nFull results: ${args.json}`);
+  }
+
+  if (differing.length === 0) {
+    console.log("\nNo differences");
+    process.exit(0);
+  }
+  console.log(`\n${differing.length} of ${results.length} requests differ`);
+  if (!prompt) {
+    console.log("Run in a terminal to review each group of differences.");
+    process.exit(1);
+  }
+  const unexpected = acknowledged.filter(group => !group.expected).length;
+  console.log(
+    unexpected === 0
+      ? `You marked all ${sorted.length} groups of differences as expected`
+      : `${unexpected} of ${sorted.length} groups of differences are not marked as expected`,
+  );
+  process.exit(unexpected === 0 ? 0 : 1);
+}
+
+await main();
diff --git a/scripts/compare-releases.sh b/scripts/compare-releases.sh
deleted file mode 100755
index 841bfaa..0000000
--- a/scripts/compare-releases.sh
+++ /dev/null
@@ -1,875 +0,0 @@
-#!/usr/bin/env bash
-
-set -uo pipefail
-
-LOCAL_BASE="${LOCAL_BASE:-http://localhost:3000}"
-PROD_BASE="${PROD_BASE:-https://api.jetkvm.com}"
-
-# Two device IDs picked so they straddle a typical staged rollout window:
-#   compare-device-1 → rollout bucket 81 (skips releases < 81%)
-#   compare-device-2 → rollout bucket 9  (catches releases >= 10%)
-# Together they exercise both eligible and ineligible rollout paths.
-DEFAULT_DEVICE_IDS=("compare-device-1" "compare-device-2")
-DEFAULT_SKUS=("__omit__" "jetkvm-v2" "jetkvm-v2-sdmmc")
-# SKUs whose responses are allowed to differ from production until the SKU
-# table is deployed there: the two Mini variants (no app artifact, no release
-# yet) and one unregistered SKU (400 locally, 404 on the old code).
-EXTRA_SKUS="${EXTRA_SKUS:-jetkvm-mini-ethernet jetkvm-mini-wireless jetkvm-v3}"
-TRISTATE_VALUES=("__omit__" "false" "true")
-
-TMP_DIR="$(mktemp -d)"
-trap 'rm -rf "$TMP_DIR"' EXIT
-
-PASS_COUNT=0
-FAIL_COUNT=0
-ACCEPTED_COUNT=0
-CASE_COUNT=0
-CASE_INDEX=0
-TOTAL_CASES=0
-PROGRESS_WIDTH=40
-
-print_usage() {
-  cat <<'EOF'
-Usage: scripts/compare-releases.sh [device_id ...]
-
-Compares release endpoint responses between:
-  - local API
-  - api.jetkvm.com
-
-Defaults:
-  LOCAL_BASE=http://localhost:3000
-  PROD_BASE=https://api.jetkvm.com
-  device_ids=(compare-device-1)
-
-Environment overrides:
-  LOCAL_BASE              Override local host
-  PROD_BASE               Override production host
-  CURL_TIMEOUT            Curl max time in seconds (default: 30)
-  CURL_CONNECT_TIMEOUT    Curl connect timeout in seconds (default: 10)
-  FAIL_FAST               Stop after first failed case (default: true)
-  EXTRA_SKUS              Space-separated SKUs compared with the expected 400-vs-404
-                          deviation accepted (default: both Mini SKUs and jetkvm-v3)
-
-Examples:
-  scripts/compare-releases.sh
-  scripts/compare-releases.sh device-a device-b
-  LOCAL_BASE=http://localhost:3001 PROD_BASE=https://api.jetkvm.com scripts/compare-releases.sh
-EOF
-}
-
-if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
-  print_usage
-  exit 0
-fi
-
-if (($# > 0)); then
-  DEVICE_IDS=("$@")
-else
-  DEVICE_IDS=("${DEFAULT_DEVICE_IDS[@]}")
-fi
-
-CURL_TIMEOUT="${CURL_TIMEOUT:-30}"
-CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-10}"
-MAX_PARALLEL="${MAX_PARALLEL:-5}"
-RETRY_COUNT="${RETRY_COUNT:-2}"
-RETRY_DELAY_SECONDS="${RETRY_DELAY_SECONDS:-1}"
-FAIL_FAST="${FAIL_FAST:-true}"
-
-log() {
-  printf '%s\n' "$*"
-}
-
-render_progress() {
-  local completed="$1"
-  local total="$2"
-  local width="${3:-$PROGRESS_WIDTH}"
-  local filled=0
-  local empty=0
-
-  if (( total > 0 )); then
-    filled=$(( completed * width / total ))
-  fi
-  empty=$(( width - filled ))
-
-  printf '%*s' "$filled" '' | tr ' ' '#'
-  printf '%*s' "$empty" ''
-}
-
-urlencode() {
-  python3 - "$1" <<'PY'
-import sys
-from urllib.parse import quote
-
-print(quote(sys.argv[1], safe=""))
-PY
-}
-
-join_query() {
-  local -n query_keys_ref=$1
-  local -n query_values_ref=$2
-  local query=""
-  local i key value encoded
-
-  for i in "${!query_keys_ref[@]}"; do
-    key="${query_keys_ref[$i]}"
-    value="${query_values_ref[$i]}"
-    [[ "$value" == "__omit__" ]] && continue
-    encoded="$(urlencode "$value")"
-    if [[ -n "$query" ]]; then
-      query+="&"
-    fi
-    query+="${key}=${encoded}"
-  done
-
-  printf '%s' "$query"
-}
-
-header_value() {
-  local file="$1"
-  local name="$2"
-  python3 - "$file" "$name" <<'PY'
-import sys
-from pathlib import Path
-
-path = Path(sys.argv[1])
-name = sys.argv[2].lower()
-value = ""
-for raw_line in path.read_text(errors="replace").splitlines():
-    line = raw_line.strip()
-    if not line or ":" not in line:
-        continue
-    key, candidate = line.split(":", 1)
-    if key.lower() == name:
-        value = candidate.strip()
-print(value)
-PY
-}
-
-normalize_body() {
-  local body_file="$1"
-  local normalized_file="$2"
-  if [[ ! -f "$body_file" ]]; then
-    : >"$normalized_file"
-    return
-  fi
-  python3 - "$body_file" "$normalized_file" <<'PY'
-import json
-import sys
-from pathlib import Path
-
-body_path = Path(sys.argv[1])
-normalized_path = Path(sys.argv[2])
-body = body_path.read_text(errors="replace")
-
-try:
-    parsed = json.loads(body)
-except Exception:
-    normalized_path.write_text(body)
-else:
-    # Resolver bookkeeping that older deploys still serialize; the device
-    # never reads it, so it is not part of the comparison.
-    ignored_suffixes = ("CachedAt", "MaxSatisfying")
-
-    def scrub(value):
-        if isinstance(value, dict):
-            return {
-                key: scrub(child)
-                for key, child in value.items()
-                if not key.endswith(ignored_suffixes)
-            }
-        if isinstance(value, list):
-            return [scrub(item) for item in value]
-        return value
-
-    normalized_path.write_text(json.dumps(scrub(parsed), indent=2, sort_keys=True) + "\n")
-PY
-}
-
-summarize_body_mismatch() {
-  local left_file="$1"
-  local right_file="$2"
-  python3 - "$left_file" "$right_file" <<'PY'
-import json
-import sys
-from pathlib import Path
-
-left_path = Path(sys.argv[1])
-right_path = Path(sys.argv[2])
-
-def load(path):
-    try:
-        return json.loads(path.read_text(errors="replace"))
-    except Exception:
-        return path.read_text(errors="replace")
-
-left = load(left_path)
-right = load(right_path)
-
-def walk(a, b, path="$"):
-    if type(a) != type(b):
-        return path, a, b
-    if isinstance(a, dict):
-        keys = sorted(set(a) | set(b))
-        for key in keys:
-            if key not in a:
-                return f"{path}.{key}", "<missing>", b[key]
-            if key not in b:
-                return f"{path}.{key}", a[key], "<missing>"
-            result = walk(a[key], b[key], f"{path}.{key}")
-            if result is not None:
-                return result
-        return None
-    if isinstance(a, list):
-        if len(a) != len(b):
-            return f"{path}.length", len(a), len(b)
-        for idx, (av, bv) in enumerate(zip(a, b)):
-            result = walk(av, bv, f"{path}[{idx}]")
-            if result is not None:
-                return result
-        return None
-    if a != b:
-        return path, a, b
-    return None
-
-result = walk(left, right)
-if result is None:
-    print("values differ")
-else:
-    path, left_value, right_value = result
-    print(f"path={path}")
-    print(f"local={json.dumps(left_value, sort_keys=True)}")
-    print(f"prod={json.dumps(right_value, sort_keys=True)}")
-PY
-}
-
-body_diff_is_version_only_not_found() {
-  local left_file="$1"
-  local right_file="$2"
-  python3 - "$left_file" "$right_file" <<'PY'
-import json
-import re
-import sys
-from pathlib import Path
-
-try:
-    left = json.loads(Path(sys.argv[1]).read_text(errors="replace"))
-    right = json.loads(Path(sys.argv[2]).read_text(errors="replace"))
-except Exception:
-    raise SystemExit(1)
-
-if not (isinstance(left, dict) and isinstance(right, dict)):
-    raise SystemExit(1)
-
-if left.get("name") != "NotFoundError" or right.get("name") != "NotFoundError":
-    raise SystemExit(1)
-
-left_keys = set(left.keys())
-right_keys = set(right.keys())
-if left_keys != {"name", "message"} or right_keys != {"name", "message"}:
-    raise SystemExit(1)
-
-# Both messages mean "no release compatible with this SKU is available";
-# the wording differs across deploys but the device sees the same 404.
-no_compat_patterns = [
-    re.compile(r'^Version .+ has no artifact for SKU "([^"]+)"$'),
-    re.compile(r'^Version .+ predates SKU support and cannot serve SKU "([^"]+)"$'),
-    re.compile(r'^SKU "([^"]+)" is not available for version .+$'),
-    re.compile(r'^No default (?:app|system|mini) release available for SKU "([^"]+)"$'),
-    re.compile(r'^No release found for type \S+ and SKU "([^"]+)"$'),
-]
-
-def canonicalize(message):
-    for pattern in no_compat_patterns:
-        match = pattern.match(message)
-        if match:
-            return f'<no-compat-release sku="{match.group(1)}">'
-    return message
-
-left_message = left.get("message", "")
-right_message = right.get("message", "")
-
-if canonicalize(left_message) == canonicalize(right_message):
-    raise SystemExit(0)
-
-raise SystemExit(1)
-PY
-}
-
-is_accepted_deviation() {
-  local query="$1"
-  local left_prefix="$2"
-  local right_prefix="$3"
-  EXTRA_SKUS="$EXTRA_SKUS" python3 - "$query" "${left_prefix}.meta" "${right_prefix}.meta" "${left_prefix}.normalized" "${right_prefix}.normalized" <<'PY'
-import json
-import os
-import sys
-from pathlib import Path
-from urllib.parse import parse_qs
-
-query, left_meta_path, right_meta_path, left_body_path, right_body_path = sys.argv[1:]
-params = parse_qs(query, keep_blank_values=True)
-
-def one(name):
-    values = params.get(name, [])
-    return values[0] if values else None
-
-def parse_meta(path):
-    data = {}
-    for line in Path(path).read_text(errors="replace").splitlines():
-        if "=" in line:
-            key, value = line.split("=", 1)
-            data[key] = value
-    return data
-
-def load_json(path):
-    try:
-        return json.loads(Path(path).read_text(errors="replace"))
-    except Exception:
-        return None
-
-left_meta = parse_meta(left_meta_path)
-right_meta = parse_meta(right_meta_path)
-left_body = load_json(left_body_path)
-right_body = load_json(right_body_path)
-
-def accept(reason):
-    print(reason)
-    raise SystemExit(0)
-
-# Accepted behavior change:
-# For SKUs in EXTRA_SKUS (unregistered, or registered but without releases or
-# without the requested artifact) the SKU table answers 400 or 404 with its
-# own wording; the code before it answered 400 or 404 from a different check.
-# Both sides refusing with a client error means the same thing here.
-if one("sku") in os.environ.get("EXTRA_SKUS", "").split():
-    def refuses(meta, body):
-        return (
-            meta.get("http_code") in ("400", "404")
-            and isinstance(body, dict)
-            and str(body.get("name", "")).endswith("Error")
-        )
-    if refuses(left_meta, left_body) and refuses(right_meta, right_body):
-        accept("SKU table: both sides refuse this SKU with a 4xx (wording and status may differ)")
-
-# Accepted behavior change:
-# Stable requests with prerelease/dev version constraints are DB-only locally.
-# Production still resolves those directly from S3. Local 404 vs prod 200 is expected.
-if one("prerelease") not in (None, "false"):
-    raise SystemExit(1)
-
-constrained_versions = [one("appVersion"), one("systemVersion")]
-has_dev_constraint = any(value and "-" in value for value in constrained_versions)
-if not has_dev_constraint:
-    raise SystemExit(1)
-
-if left_meta.get("http_code") != "404" or right_meta.get("http_code") != "200":
-    raise SystemExit(1)
-
-if not isinstance(left_body, dict) or left_body.get("name") != "NotFoundError":
-    raise SystemExit(1)
-
-if not isinstance(right_body, dict) or not right_body.get("appVersion") or not right_body.get("systemVersion"):
-    raise SystemExit(1)
-
-accept("stable dev/prerelease version constraints are DB-only locally")
-PY
-}
-
-curl_capture() {
-  local base_url="$1"
-  local path="$2"
-  local query="$3"
-  local prefix="$4"
-  local url="${base_url}${path}"
-  local headers_file="${prefix}.headers"
-  local body_file="${prefix}.body"
-  local meta_file="${prefix}.meta"
-  local stderr_file="${prefix}.stderr"
-  local exit_file="${prefix}.exit"
-  local attempt=0
-  local curl_exit=0
-  local http_code=""
-
-  if [[ -n "$query" ]]; then
-    url="${url}?${query}"
-  fi
-
-  while :; do
-    : >"$headers_file"
-    : >"$body_file"
-    : >"$meta_file"
-    : >"$stderr_file"
-
-    curl_exit=0
-    curl \
-      --silent \
-      --show-error \
-      --connect-timeout "$CURL_CONNECT_TIMEOUT" \
-      --max-time "$CURL_TIMEOUT" \
-      --dump-header "$headers_file" \
-      --output "$body_file" \
-      --write-out "http_code=%{http_code}\ncontent_type=%{content_type}\n" \
-      "$url" >"$meta_file" 2>"$stderr_file" || curl_exit=$?
-
-    printf '%s\n' "$curl_exit" >"$exit_file"
-    http_code="$(sed -n 's/^http_code=//p' "$meta_file")"
-
-    if (( curl_exit == 0 )) && [[ ! "$http_code" =~ ^52[0-9]$ ]]; then
-      break
-    fi
-
-    if (( attempt >= RETRY_COUNT )); then
-      break
-    fi
-
-    attempt=$((attempt + 1))
-    sleep "$RETRY_DELAY_SECONDS"
-  done
-}
-
-compare_scalar_files() {
-  local label="$1"
-  local left_file="$2"
-  local right_file="$3"
-  local left_value right_value
-
-  left_value="$(tr -d '\r' <"$left_file")"
-  right_value="$(tr -d '\r' <"$right_file")"
-  if [[ "$left_value" == "$right_value" ]]; then
-    return 1
-  fi
-
-  printf '%s\n' "$label"
-  printf '    local=%s\n' "${left_value:-<empty>}"
-  printf '    prod=%s\n' "${right_value:-<empty>}"
-  return 0
-}
-
-summarize_meta_mismatch() {
-  local left_file="$1"
-  local right_file="$2"
-  python3 - "$left_file" "$right_file" <<'PY'
-import sys
-from pathlib import Path
-
-def parse(path_str):
-    data = {}
-    for line in Path(path_str).read_text(errors="replace").splitlines():
-        if "=" not in line:
-            continue
-        key, value = line.split("=", 1)
-        data[key] = value
-    return data
-
-left = parse(sys.argv[1])
-right = parse(sys.argv[2])
-keys = sorted(set(left) | set(right))
-for key in keys:
-    if left.get(key) != right.get(key):
-        print(f"{key}")
-        print(f"local={left.get(key, '<missing>')}")
-        print(f"prod={right.get(key, '<missing>')}")
-PY
-}
-
-write_case_result() {
-  local result_file="$1"
-  local case_name="$2"
-  local path="$3"
-  local query="$4"
-  local left_prefix="$5"
-  local right_prefix="$6"
-  local left_norm="${left_prefix}.normalized"
-  local right_norm="${right_prefix}.normalized"
-  local left_location right_location
-  local failed=0
-  local details=""
-  local mismatch_count=0
-  local output=""
-  local accepted_reason=""
-
-  normalize_body "${left_prefix}.body" "$left_norm"
-  normalize_body "${right_prefix}.body" "$right_norm"
-
-  if output="$(compare_scalar_files "exit-code mismatch" "${left_prefix}.exit" "${right_prefix}.exit")"; then
-    mismatch_count=$((mismatch_count + 1))
-    details+="$output"$'\n'
-    failed=1
-  fi
-
-  if output="$(summarize_meta_mismatch "${left_prefix}.meta" "${right_prefix}.meta")" && [[ -n "$output" ]]; then
-    mismatch_count=$((mismatch_count + 1))
-    details+=$'  response-meta mismatch\n'
-    details+="$(printf '%s\n' "$output" | sed 's/^/    /')"$'\n'
-    failed=1
-  fi
-
-  left_location="$(header_value "${left_prefix}.headers" "location")"
-  right_location="$(header_value "${right_prefix}.headers" "location")"
-  if [[ "$left_location" != "$right_location" ]]; then
-    mismatch_count=$((mismatch_count + 1))
-    details+=$'  location mismatch\n'
-    details+="    local=${left_location:-<none>}"$'\n'
-    details+="    prod=${right_location:-<none>}"$'\n'
-    failed=1
-  fi
-
-  if [[ -s "${left_prefix}.body" || -s "${right_prefix}.body" ]]; then
-    if ! cmp -s "$left_norm" "$right_norm"; then
-      if body_diff_is_version_only_not_found "$left_norm" "$right_norm"; then
-        :
-      else
-        mismatch_count=$((mismatch_count + 1))
-        details+=$'  body mismatch\n'
-        details+="$(summarize_body_mismatch "$left_norm" "$right_norm" | sed 's/^/    /')"$'\n'
-        failed=1
-      fi
-    fi
-  fi
-
-  if (( failed == 1 )) && accepted_reason="$(is_accepted_deviation "$query" "$left_prefix" "$right_prefix")"; then
-    failed=0
-    details=""
-    mismatch_count=0
-  else
-    accepted_reason=""
-  fi
-
-  {
-    printf 'status=%s\n' "$([[ $failed -eq 0 ]] && { [[ -n "$accepted_reason" ]] && printf accepted || printf pass; } || printf fail)"
-    printf 'case_name=%s\n' "$case_name"
-    printf 'path=%s\n' "$path"
-    printf 'query=%s\n' "$query"
-    printf 'accepted_reason=%s\n' "$accepted_reason"
-    printf 'mismatch_count=%s\n' "$mismatch_count"
-    printf 'details<<__DETAILS__\n%s__DETAILS__\n' "$details"
-    printf 'local_stderr<<__STDERR__\n%s__STDERR__\n' "$(tr '\n' ' ' <"${left_prefix}.stderr")"
-    printf 'prod_stderr<<__STDERR__\n%s__STDERR__\n' "$(tr '\n' ' ' <"${right_prefix}.stderr")"
-  } >"$result_file"
-}
-
-run_case_worker() {
-  local case_name="$1"
-  local path="$2"
-  local query="$3"
-  local safe_case="$4"
-  local result_file="$5"
-
-  local local_prefix="$TMP_DIR/${safe_case}.local"
-  local prod_prefix="$TMP_DIR/${safe_case}.prod"
-
-  curl_capture "$LOCAL_BASE" "$path" "$query" "$local_prefix" &
-  local local_pid=$!
-  curl_capture "$PROD_BASE" "$path" "$query" "$prod_prefix" &
-  local prod_pid=$!
-  wait "$local_pid"
-  wait "$prod_pid"
-
-  write_case_result "$result_file" "$case_name" "$path" "$query" "$local_prefix" "$prod_prefix"
-}
-
-print_case_result() {
-  local result_file="$1"
-  local progress_bar
-  local status case_name path query accepted_reason mismatch_count details local_stderr prod_stderr
-
-  progress_bar="$(render_progress "$CASE_INDEX" "$TOTAL_CASES")"
-  status="$(sed -n 's/^status=//p' "$result_file")"
-  case_name="$(sed -n 's/^case_name=//p' "$result_file")"
-  path="$(sed -n 's/^path=//p' "$result_file")"
-  query="$(sed -n 's/^query=//p' "$result_file")"
-  accepted_reason="$(sed -n 's/^accepted_reason=//p' "$result_file")"
-  mismatch_count="$(sed -n 's/^mismatch_count=//p' "$result_file")"
-  details="$(awk '/^details<<__DETAILS__/{flag=1;next}/^__DETAILS__$/{flag=0}flag' "$result_file")"
-  local_stderr="$(awk '/^local_stderr<<__STDERR__/{flag=1;next}/^__STDERR__$/{if(flag){flag=0; exit}}flag' "$result_file")"
-  prod_stderr="$(awk 'found && /^__STDERR__$/ {exit} /^prod_stderr<<__STDERR__$/ {found=1; next} found {print}' "$result_file")"
-
-  if [[ "$status" == "pass" ]]; then
-    PASS_COUNT=$((PASS_COUNT + 1))
-    printf '\r[%s] %4d/%-4d | pass:%d fail:%d' \
-      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$FAIL_COUNT"
-    if (( CASE_INDEX == TOTAL_CASES )); then
-      printf '\n'
-    fi
-  elif [[ "$status" == "accepted" ]]; then
-    ACCEPTED_COUNT=$((ACCEPTED_COUNT + 1))
-    printf '\r\033[K'
-    printf '[%s] %4d/%-4d | pass:%d accepted:%d fail:%d\n' \
-      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$ACCEPTED_COUNT" "$FAIL_COUNT"
-    printf '  ACCEPT %s\n' "$case_name"
-    printf '  %s%s%s\n' "$path" "${query:+?$query}" ""
-    printf '  %s\n' "$accepted_reason"
-  else
-    FAIL_COUNT=$((FAIL_COUNT + 1))
-    printf '\r\033[K'
-    printf '[%s] %4d/%-4d | pass:%d fail:%d\n' \
-      "$progress_bar" "$CASE_INDEX" "$TOTAL_CASES" "$PASS_COUNT" "$FAIL_COUNT"
-    printf '  FAIL %s\n' "$case_name"
-    printf '  %s%s%s\n' "$path" "${query:+?$query}" ""
-    printf '%s' "$details"
-    if [[ -n "$local_stderr" || -n "$prod_stderr" ]]; then
-      printf '  stderr\n'
-      printf '    local=%s\n' "$local_stderr"
-      printf '    prod=%s\n' "$prod_stderr"
-    fi
-    if [[ "${mismatch_count:-0}" == "0" ]]; then
-      printf '  mismatch detected\n'
-    fi
-  fi
-}
-
-stop_requested() {
-  [[ "$FAIL_FAST" != "false" && "$FAIL_COUNT" -gt 0 ]]
-}
-
-wait_for_one_job() {
-  local pid done_pid result_file
-  while :; do
-    for pid in "${!JOB_RESULT_FILES[@]}"; do
-      if ! kill -0 "$pid" 2>/dev/null; then
-        wait "$pid" || true
-        done_pid="$pid"
-        result_file="${JOB_RESULT_FILES[$pid]}"
-        unset "JOB_RESULT_FILES[$pid]"
-        CASE_INDEX=$((CASE_INDEX + 1))
-        print_case_result "$result_file"
-        rm -f "$result_file"
-        return
-      fi
-    done
-    sleep 0.05
-  done
-}
-
-drain_jobs() {
-  while ((${#JOB_RESULT_FILES[@]} > 0)); do
-    wait_for_one_job
-    if stop_requested; then
-      for pid in "${!JOB_RESULT_FILES[@]}"; do
-        kill "$pid" 2>/dev/null || true
-      done
-      JOB_RESULT_FILES=()
-      break
-    fi
-  done
-}
-
-extract_versions() {
-  local device_id="$1"
-  local prerelease="$2"
-  local sku="$3"
-  local prefix="$4"
-
-  local query_keys=("deviceId" "prerelease" "sku")
-  local query_values=("$device_id" "$prerelease" "$sku")
-  local query
-  query="$(join_query query_keys query_values)"
-
-  curl_capture "$PROD_BASE" "/releases" "$query" "$prefix"
-
-  python3 - "$prefix.body" <<'PY'
-import json
-import sys
-from pathlib import Path
-
-path = Path(sys.argv[1])
-try:
-    payload = json.loads(path.read_text(errors="replace"))
-except Exception:
-    print("")
-    print("")
-    raise SystemExit(0)
-
-print(payload.get("appVersion", ""))
-print(payload.get("systemVersion", ""))
-PY
-}
-
-build_value_set() {
-  local exact_version="$1"
-  local prerelease_version="$2"
-  local values=("__omit__" "*")
-
-  if [[ -n "$exact_version" ]]; then
-    values+=("$exact_version")
-  fi
-  if [[ -n "$prerelease_version" && "$prerelease_version" != "$exact_version" ]]; then
-    values+=("$prerelease_version")
-  fi
-
-  printf '%s\n' "${values[@]}" | awk '!seen[$0]++'
-}
-
-run_case() {
-  local case_name="$1"
-  local path="$2"
-  local -n case_keys_ref=$3
-  local -n case_values_ref=$4
-  local query
-
-  CASE_COUNT=$((CASE_COUNT + 1))
-  query="$(join_query case_keys_ref case_values_ref)"
-
-  local safe_case
-  safe_case="$(printf '%s' "$case_name" | tr ' /?=&' '_____')"
-  local result_file="$TMP_DIR/${safe_case}.result"
-
-  run_case_worker "$case_name" "$path" "$query" "$safe_case" "$result_file" &
-  JOB_RESULT_FILES[$!]="$result_file"
-
-  while ((${#JOB_RESULT_FILES[@]} >= MAX_PARALLEL)); do
-    wait_for_one_job
-    if stop_requested; then
-      return
-    fi
-  done
-}
-
-log "Comparing release endpoints"
-log "  local:   $LOCAL_BASE"
-log "  prod:    $PROD_BASE"
-log "  deviceIds: ${DEVICE_IDS[*]}"
-
-mapfile -t stable_versions < <(extract_versions "${DEVICE_IDS[0]}" "__omit__" "__omit__" "$TMP_DIR/baseline-stable")
-mapfile -t prerelease_versions < <(extract_versions "${DEVICE_IDS[0]}" "true" "__omit__" "$TMP_DIR/baseline-prerelease")
-
-STABLE_APP_VERSION="${stable_versions[0]:-}"
-STABLE_SYSTEM_VERSION="${stable_versions[1]:-}"
-PRERELEASE_APP_VERSION="${prerelease_versions[0]:-}"
-PRERELEASE_SYSTEM_VERSION="${prerelease_versions[1]:-}"
-
-# Range constraints around the live stable version: the caret range resolves
-# to the newest patch of the current minor, "<stable" to the newest release
-# before it, which for app/system reaches back into the pre-skus/ layout.
-range_values() {
-  local version="$1"
-  [[ -z "$version" ]] && return
-  local major minor
-  IFS=. read -r major minor _ <<<"$version"
-  printf '%s\n' "^${major}.${minor}.0" "<${version}"
-}
-mapfile -t APP_RANGE_VALUES < <(range_values "$STABLE_APP_VERSION")
-mapfile -t SYSTEM_RANGE_VALUES < <(range_values "$STABLE_SYSTEM_VERSION")
-read -r -a EXTRA_SKU_LIST <<<"$EXTRA_SKUS"
-
-mapfile -t APP_VERSION_VALUES < <(build_value_set "$STABLE_APP_VERSION" "$PRERELEASE_APP_VERSION")
-mapfile -t SYSTEM_VERSION_VALUES < <(build_value_set "$STABLE_SYSTEM_VERSION" "$PRERELEASE_SYSTEM_VERSION")
-
-TOTAL_CASES=$(( ${#DEVICE_IDS[@]} * ${#TRISTATE_VALUES[@]} * ${#APP_VERSION_VALUES[@]} * ${#SYSTEM_VERSION_VALUES[@]} * ${#DEFAULT_SKUS[@]} + ${#TRISTATE_VALUES[@]} * ${#DEFAULT_SKUS[@]} * 2 + ${#DEVICE_IDS[@]} * ${#DEFAULT_SKUS[@]} * (${#APP_RANGE_VALUES[@]} + ${#SYSTEM_RANGE_VALUES[@]}) + ${#EXTRA_SKU_LIST[@]} * 4 ))
-declare -A JOB_RESULT_FILES=()
-log "  total cases: $TOTAL_CASES"
-log "  parallel: $MAX_PARALLEL"
-log "  failFast: $FAIL_FAST"
-log
-
-for device_id in "${DEVICE_IDS[@]}"; do
-  for prerelease in "${TRISTATE_VALUES[@]}"; do
-    for app_version in "${APP_VERSION_VALUES[@]}"; do
-      for system_version in "${SYSTEM_VERSION_VALUES[@]}"; do
-        for sku in "${DEFAULT_SKUS[@]}"; do
-          if stop_requested; then
-            break 5
-          fi
-          query_keys=("deviceId" "prerelease" "appVersion" "systemVersion" "sku")
-          query_values=("$device_id" "$prerelease" "$app_version" "$system_version" "$sku")
-          run_case \
-            "GET /releases deviceId=$device_id prerelease=$prerelease appVersion=$app_version systemVersion=$system_version sku=$sku" \
-            "/releases" \
-            query_keys \
-            query_values
-        done
-      done
-    done
-  done
-done
-
-for prerelease in "${TRISTATE_VALUES[@]}"; do
-  for sku in "${DEFAULT_SKUS[@]}"; do
-    if stop_requested; then
-      break 2
-    fi
-    query_keys=("prerelease" "sku")
-    query_values=("$prerelease" "$sku")
-    run_case \
-      "GET /releases/app/latest prerelease=$prerelease sku=$sku" \
-      "/releases/app/latest" \
-      query_keys \
-      query_values
-    run_case \
-      "GET /releases/system_recovery/latest prerelease=$prerelease sku=$sku" \
-      "/releases/system_recovery/latest" \
-      query_keys \
-      query_values
-  done
-done
-
-# Range and older-version constraints, one axis at a time.
-for device_id in "${DEVICE_IDS[@]}"; do
-  for sku in "${DEFAULT_SKUS[@]}"; do
-    for app_version in "${APP_RANGE_VALUES[@]}"; do
-      if stop_requested; then
-        break 3
-      fi
-      query_keys=("deviceId" "appVersion" "sku")
-      query_values=("$device_id" "$app_version" "$sku")
-      run_case \
-        "GET /releases deviceId=$device_id appVersion=$app_version sku=$sku" \
-        "/releases" \
-        query_keys \
-        query_values
-    done
-    for system_version in "${SYSTEM_RANGE_VALUES[@]}"; do
-      if stop_requested; then
-        break 3
-      fi
-      query_keys=("deviceId" "systemVersion" "sku")
-      query_values=("$device_id" "$system_version" "$sku")
-      run_case \
-        "GET /releases deviceId=$device_id systemVersion=$system_version sku=$sku" \
-        "/releases" \
-        query_keys \
-        query_values
-    done
-  done
-done
-
-# SKUs the SKU table treats differently from the code before it.
-for sku in "${EXTRA_SKU_LIST[@]}"; do
-  for prerelease in "__omit__" "true"; do
-    if stop_requested; then
-      break 2
-    fi
-    query_keys=("deviceId" "prerelease" "sku")
-    query_values=("${DEVICE_IDS[0]}" "$prerelease" "$sku")
-    run_case \
-      "GET /releases deviceId=${DEVICE_IDS[0]} prerelease=$prerelease sku=$sku" \
-      "/releases" \
-      query_keys \
-      query_values
-  done
-  query_keys=("sku")
-  query_values=("$sku")
-  run_case \
-    "GET /releases/app/latest sku=$sku" \
-    "/releases/app/latest" \
-    query_keys \
-    query_values
-  run_case \
-    "GET /releases/system_recovery/latest sku=$sku" \
-    "/releases/system_recovery/latest" \
-    query_keys \
-    query_values
-done
-
-drain_jobs
-
-log
-log "Summary"
-log "  cases:  $CASE_COUNT"
-log "  pass:   $PASS_COUNT"
-log "  accept: $ACCEPTED_COUNT"
-log "  fail:   $FAIL_COUNT"
-
-if ((FAIL_COUNT > 0)); then
-  exit 1
-fi
diff --git a/src/auth.ts b/src/auth.ts
index 96c895d..6ddb9e9 100644
--- a/src/auth.ts
+++ b/src/auth.ts
@@ -38,7 +38,8 @@ export const verifyToken = async (idToken: string) => {
   }
 };
 
-export const authenticated = async (req: Request, res: Response, next: NextFunction) => {
+/** The verified ID token of the session's user; throws UnauthorizedError otherwise. */
+export const verifySession = async (req: Request) => {
   const idToken = req.session?.id_token;
   if (!idToken) throw new UnauthorizedError();
 
@@ -55,6 +56,11 @@ export const authenticated = async (req: Request, res: Response, next: NextFunct
     throw new UnauthorizedError("Account is not in the allowlist", "account_not_allowed");
   }
 
+  return payload;
+};
+
+export const authenticated = async (req: Request, res: Response, next: NextFunction) => {
+  await verifySession(req);
   next();
 };
 
diff --git a/src/devices.ts b/src/devices.ts
index b6d54d4..b30ebd4 100644
--- a/src/devices.ts
+++ b/src/devices.ts
@@ -8,7 +8,7 @@ import {
   UnprocessableEntityError,
 } from "./errors";
 import * as crypto from "crypto";
-import { authenticated } from "./auth";
+import { verifySession } from "./auth";
 import { activeConnections } from "./webrtc-signaling";
 import { effectiveSku } from "./skus";
 
@@ -124,19 +124,8 @@ export const Delete = async (
     return res.status(204).send();
   }
 
-  // If the user doesn't have a secret token, we check their session cookie
-  try {
-    await new Promise<void>(resolve => {
-      authenticated(req, res, () => {
-        resolve();
-      });
-    });
-  } catch (error) {
-    throw new BadRequestError("Unauthorized");
-  }
-
-  const idToken = req.session?.id_token;
-  const { sub } = jose.decodeJwt(idToken);
+  // Without a device token, only a signed-in owner may delete the device
+  const { sub } = await verifySession(req);
   if (!sub) throw new UnauthorizedError("Missing sub in token");
 
   const { id } = req.params;
diff --git a/src/releases.ts b/src/releases.ts
index bc10035..5ff4ccd 100644
--- a/src/releases.ts
+++ b/src/releases.ts
@@ -147,7 +147,7 @@ const objectExists = (key: string) => s3ObjectExists(s3Client, bucketName, key);
  *   - Fails if the requested SKU is not available
  *
  * For legacy versions (no skus/ folder):
- *   - Returns legacy path for SKUs that predate the SKU layout
+ *   - Returns legacy path for SKUs that predate the SKU layout, if the file exists
  *   - Fails for every other SKU because legacy firmware predates
  *     that hardware and may not be compatible
  *
@@ -177,8 +177,9 @@ async function resolveArtifactPath(
   // For legacy versions (pre-SKU folder structure), we only serve the SKUs
   // that predate the layout. This prevents newer hardware variants from
   // rolling back to old firmware that may not have compatible binaries.
-  if (legacyCompatibleSkus(prefix).includes(sku)) {
-    return `${prefix}/${version}/${file}`;
+  const legacyPath = `${prefix}/${version}/${file}`;
+  if (legacyCompatibleSkus(prefix).includes(sku) && (await objectExists(legacyPath))) {
+    return legacyPath;
   }
 
   throw noArtifactForSku(version, sku);
@@ -238,66 +239,77 @@ async function enrichWithSigUrls(release: Release, sku: string): Promise<void> {
   );
 }
 
+/** Versions checked, newest first, before newestVersionWithArtifact gives up. */
+const MAX_FALLBACK_VERSIONS = 5;
+
+/**
+ * The newest version under the artifact's prefix that satisfies the range and
+ * has the artifact for the SKU, and that artifact's path. A version without a
+ * build for the SKU is skipped, as the stable path skips releases without a
+ * compatible artifact, so a SKU missing from one upload does not hide older ones.
+ * Only the newest `maxVersions` matching versions are checked.
+ */
+async function newestVersionWithArtifact(
+  { prefix, file }: Artifact,
+  sku: string,
+  range: string,
+  includePrerelease: boolean,
+  maxVersions = MAX_FALLBACK_VERSIONS,
+): Promise<{ version: string; path: string }> {
+  const response = await s3Client.send(
+    new ListObjectsV2Command({ Bucket: bucketName, Prefix: `${prefix}/`, Delimiter: "/" }),
+  );
+
+  const versions = (response.CommonPrefixes ?? [])
+    .map(cp => cp.Prefix!.split("/")[1])
+    .filter(v => v && semver.valid(v));
+  if (versions.length === 0) {
+    throw new NotFoundError(`No versions found under prefix ${prefix}`);
+  }
+
+  const candidates = semver.rsort(
+    versions.filter(v => semver.satisfies(v, range, { includePrerelease })),
+  );
+  if (candidates.length === 0) {
+    throw new NotFoundError(`No version found under prefix ${prefix} that satisfies ${range}`);
+  }
+
+  // Each version costs one or two S3 calls and a miss is not cached, so the
+  // walk is short: a version without the SKU's build is an upload mistake,
+  // not something that runs for many versions.
+  for (const version of candidates.slice(0, maxVersions)) {
+    try {
+      return { version, path: await resolveArtifactPath(prefix, version, sku, file) };
+    } catch (error) {
+      if (!(error instanceof NotFoundError)) throw error;
+    }
+  }
+  throw noArtifactForSku(candidates[0], sku);
+}
+
 async function getLatestVersion(
   artifact: Artifact,
   includePrerelease: boolean,
   maxSatisfying: string = "*",
   sku: string,
 ): Promise<ReleaseMetadata> {
-  const { prefix } = artifact;
-  const cacheKey = `${prefix}-${includePrerelease}-${maxSatisfying}-${sku}`;
+  const cacheKey = `${artifact.prefix}-${includePrerelease}-${maxSatisfying}-${sku}`;
   const cached = releaseCache.get(cacheKey);
   if (cached) return cached;
 
-  const listCommand = new ListObjectsV2Command({
-    Bucket: bucketName,
-    Prefix: prefix + "/",
-    Delimiter: "/",
-  });
-
-  const response = await s3Client.send(listCommand);
-
-  if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) {
-    throw new NotFoundError(`No versions found under prefix ${prefix}`);
-  }
-
-  // Extract version folder names
-  let versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1])
-    .filter(Boolean)
-    .filter(v => semver.valid(v));
-
-  if (versions.length === 0) {
-    throw new NotFoundError(`No valid versions found under prefix ${prefix}`);
-  }
-
-  // Get the latest version, optionally including prerelease versions
-  const latestVersion = semver.maxSatisfying(versions, maxSatisfying, {
+  const { version, path } = await newestVersionWithArtifact(
+    artifact,
+    sku,
+    maxSatisfying,
     includePrerelease,
-  }) as string;
-  if (!latestVersion) {
-    throw new NotFoundError(
-      `No version found under prefix ${prefix} that satisfies ${maxSatisfying}`,
-    );
-  }
-
-  const selectedPath = await resolveArtifactPath(prefix, latestVersion, sku, artifact.file);
-  const url = `${baseUrl}/${selectedPath}`;
+  );
 
   const hashResponse = await s3Client.send(
-    new GetObjectCommand({
-      Bucket: bucketName,
-      Key: `${selectedPath}.sha256`,
-    }),
+    new GetObjectCommand({ Bucket: bucketName, Key: `${path}.sha256` }),
   );
-
   const hash = await streamToString(hashResponse.Body);
 
-  // Cache the release metadata
-  const release: ReleaseMetadata = {
-    version: latestVersion,
-    url,
-    hash,
-  };
+  const release: ReleaseMetadata = { version, url: `${baseUrl}/${path}`, hash };
   releaseCache.set(cacheKey, release);
   return release;
 }
@@ -509,6 +521,34 @@ async function getReleaseByRange(
   return latestRelease;
 }
 
+/**
+ * The release a background update check offers: the newest release that is
+ * rolling out, if the device is in its bucket and it has a build for the SKU,
+ * else the default (the newest 100% release with a build for the SKU). A
+ * staged release without a build for the SKU (e.g. a SKU-specific build has
+ * not shipped yet) keeps the default rather than 404 the whole request.
+ */
+async function rolledOutRelease(
+  prefix: string,
+  sku: string,
+  deviceId: string,
+): Promise<ReleaseMetadata> {
+  // Sequential on purpose: when the prefix has no releases at all, the 404
+  // from the latest lookup must win over the 500 from the default lookup.
+  const latest = await getLatestRelease(prefix, sku);
+  const fallback = await getDefaultRelease(prefix, sku);
+
+  const useLatest =
+    latest.artifacts.length > 0 &&
+    (await isDeviceEligibleForLatestRelease(latest.rolloutPercentage, deviceId));
+
+  const chosen = useLatest ? latest : fallback;
+  if (!chosen) {
+    throw new NotFoundError(`No ${prefix} release is rolled out yet for SKU "${sku}"`);
+  }
+  return dbReleaseToMetadata(chosen, sku);
+}
+
 export async function Retrieve(req: Request, res: Response) {
   const query = parseQuery(retrieveQuerySchema, req);
   const { sku, deviceId } = query;
@@ -518,7 +558,6 @@ export async function Retrieve(req: Request, res: Response) {
     app: toSemverRange(query.appVersion),
     system: toSemverRange(query.systemVersion),
   };
-  const skipRollout = artifacts.some(({ kind }) => constraints[kind] !== "*");
 
   // Prereleases are not imported into the DB by the stable sync script.
   if (query.prerelease) {
@@ -537,45 +576,17 @@ export async function Retrieve(req: Request, res: Response) {
     return res.json(remoteRelease);
   }
 
-  // Version-constrained stable requests skip rollout but still read DB metadata.
-  if (skipRollout) {
-    const constrained: OtaReleases = {};
-    for (const { kind, prefix } of artifacts) {
-      constrained[kind] = dbReleaseToMetadata(
-        await getReleaseByRange(prefix, sku, constraints[kind]),
-        sku,
-      );
-    }
-    const responseJson = toRelease(constrained);
-    await addStableSigUrls(responseJson);
-    return res.json(responseJson);
-  }
-
-  // Background update checks follow rollout percentages so new releases roll
-  // out gradually. Devices outside the bucket fall back to the default (the
-  // newest 100%-rolled-out release). If the latest release lacks a compatible
-  // artifact for this SKU (e.g. a SKU-specific build hasn't shipped yet) we
-  // silently keep the default rather than 404 the whole request.
+  // A pinned part (appVersion, systemVersion) gets the newest release in its
+  // range, rolled out or not. Every other part follows the rollout, as a
+  // background update check does, so pinning one part never offers an
+  // unreleased build of the other.
   const offered: OtaReleases = {};
   await Promise.all(
     artifacts.map(async ({ kind, prefix }) => {
-      // Sequential on purpose: when the prefix has no releases at all, the
-      // 404 from the latest lookup must win over the 500 from the default
-      // lookup, which the parallel form left to query timing.
-      const latest = await getLatestRelease(prefix, sku);
-      const fallback = await getDefaultRelease(prefix, sku);
-
-      const useLatest =
-        latest.artifacts.length > 0 &&
-        (await isDeviceEligibleForLatestRelease(latest.rolloutPercentage, deviceId));
-
-      const chosen = useLatest ? latest : fallback;
-      if (!chosen) {
-        throw new NotFoundError(
-          `No ${prefix} release is rolled out yet for SKU "${sku}"`,
-        );
-      }
-      offered[kind] = dbReleaseToMetadata(chosen, sku);
+      offered[kind] =
+        constraints[kind] === "*"
+          ? await rolledOutRelease(prefix, sku, deviceId)
+          : dbReleaseToMetadata(await getReleaseByRange(prefix, sku, constraints[kind]), sku);
     }),
   );
 
@@ -611,7 +622,8 @@ function releaseCacheKey(prefix: string, query: LatestQuery): string {
 /**
  * 302 to the newest recovery image for the requested SKU. The product table
  * says which prefix holds it and what it is called. The route keeps its
- * historical name.
+ * historical name. Only the newest version counts: if it has no image for the
+ * SKU, the answer is 404 rather than an older image.
  */
 export const RetrieveLatestSystemRecovery = cachedRedirect(
   query => releaseCacheKey("recovery", query),
@@ -623,43 +635,14 @@ export const RetrieveLatestSystemRecovery = cachedRedirect(
       );
     }
 
-    const listCommand = new ListObjectsV2Command({
-      Bucket: bucketName,
-      Prefix: `${recovery.prefix}/`,
-      Delimiter: "/",
-    });
-    const response = await s3Client.send(listCommand);
-
-    // Extract version folder names
-    if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) {
-      throw new NotFoundError(`No versions found under prefix ${recovery.prefix} for recovery`);
-    }
-
-    // Get the latest version
-    const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1])
-      .filter(Boolean)
-      .filter(v => semver.valid(v));
-
-    const latestVersion = semver.maxSatisfying(versions, "*", {
-      includePrerelease: query.prerelease,
-    }) as string;
-
-    if (!latestVersion) {
-      throw new NotFoundError(`No valid ${recovery.prefix} recovery versions found`);
-    }
-
-    const artifactPath = await resolveArtifactPath(
-      recovery.prefix,
-      latestVersion,
+    const { path } = await newestVersionWithArtifact(
+      recovery,
       query.sku,
-      recovery.file,
+      "*",
+      query.prerelease,
+      1,
     );
-
-    if (!(await objectExists(artifactPath))) {
-      throw new NotFoundError(`Recovery image not found for version ${latestVersion}`);
-    }
-
-    return `${baseUrl}/${artifactPath}`;
+    return `${baseUrl}/${path}`;
   },
 );
 
@@ -670,7 +653,9 @@ export const RetrieveLatestSystemRecovery = cachedRedirect(
  * object only has to exist. The product table
  * says which prefix holds it, so the same URL serves every product. Used by
  * build tooling (rv1106-system pulls the app binary into the system image)
- * and by flashing scripts.
+ * and by flashing scripts. Only the newest version counts: if it has no build
+ * for the SKU, the answer is 404, so a build fails instead of silently
+ * packaging an older one.
  */
 function latestArtifactRedirect(kind: OtaKind) {
   return cachedRedirect(
@@ -680,43 +665,15 @@ function latestArtifactRedirect(kind: OtaKind) {
       if (!artifact) {
         throw new BadRequestError(`SKU "${query.sku}" has no ${kind} artifact`);
       }
-      const { prefix } = artifact;
-
-      const listCommand = new ListObjectsV2Command({
-        Bucket: bucketName,
-        Prefix: `${prefix}/`,
-        Delimiter: "/",
-      });
-      const response = await s3Client.send(listCommand);
-
-      if (!response.CommonPrefixes || response.CommonPrefixes.length === 0) {
-        throw new NotFoundError(`No ${prefix} versions found`);
-      }
-
-      const versions = response.CommonPrefixes.map(cp => cp.Prefix!.split("/")[1]).filter(
-        v => semver.valid(v),
-      );
 
-      const latestVersion = semver.maxSatisfying(versions, "*", {
-        includePrerelease: query.prerelease,
-      }) as string;
-
-      if (!latestVersion) {
-        throw new NotFoundError(`No valid ${prefix} versions found`);
-      }
-
-      const artifactPath = await resolveArtifactPath(
-        prefix,
-        latestVersion,
+      const { path } = await newestVersionWithArtifact(
+        artifact,
         query.sku,
-        artifact.file,
+        "*",
+        query.prerelease,
+        1,
       );
-
-      if (!(await objectExists(artifactPath))) {
-        throw new NotFoundError(`${prefix} artifact not found for version ${latestVersion}`);
-      }
-
-      return `${baseUrl}/${artifactPath}`;
+      return `${baseUrl}/${path}`;
     },
   );
 }
diff --git a/src/webrtc-signaling.ts b/src/webrtc-signaling.ts
index c29f788..49ef4d2 100644
--- a/src/webrtc-signaling.ts
+++ b/src/webrtc-signaling.ts
@@ -33,10 +33,13 @@ export const iceServers = toICEServers(
 );
 
 // Helper function to update device last seen timestamp
+// Runs from socket event handlers, where a rejection would be unhandled
 async function updateDeviceLastSeen(id: string) {
-  const device = await prisma.device.findUnique({ where: { id } });
-  if (device) {
-    return prisma.device.update({ where: { id }, data: { lastSeen: new Date() } });
+  try {
+    // updateMany: the device may have been deleted while it disconnected
+    await prisma.device.updateMany({ where: { id }, data: { lastSeen: new Date() } });
+  } catch (error) {
+    console.error(`[Device] Could not record last seen for ${id}:`, error);
   }
 }
 
@@ -48,18 +51,24 @@ export function registerWebSocketRouter(
   server: Server<typeof IncomingMessage, typeof ServerResponse>,
 ) {
   server.on("upgrade", async (req: IncomingMessage, socket: Socket, head: Buffer) => {
-    const url = new URL(req.url || "", "http://localhost"); // We don't care about the hostname, we're just using the path to route
-    const path = url.pathname;
-
-    // Route to appropriate handler based on path
-    // This path should be something like /webrtc/signaling/device, but due to legacy reasons we have to use `/` for device ws regitstrations
-    if (path === "/") {
-      await handleDeviceSocketRequest(req, socket, head);
-    } else if (path === "/webrtc/signaling/client") {
-      await handleClientSocketRequest(req, socket, head);
-    } else {
-      console.log(`[Webrtc] Unrecognized path: ${path}`);
-      return socket.destroy();
+    // An event listener: nothing awaits it, so it must not reject
+    try {
+      const url = new URL(req.url || "", "http://localhost"); // We don't care about the hostname, we're just using the path to route
+      const path = url.pathname;
+
+      // Route to appropriate handler based on path
+      // This path should be something like /webrtc/signaling/device, but due to legacy reasons we have to use `/` for device ws regitstrations
+      if (path === "/") {
+        await handleDeviceSocketRequest(req, socket, head);
+      } else if (path === "/webrtc/signaling/client") {
+        await handleClientSocketRequest(req, socket, head);
+      } else {
+        console.log(`[Webrtc] Unrecognized path: ${path}`);
+        socket.destroy();
+      }
+    } catch (error) {
+      console.error("[Webrtc] Error handling upgrade request:", error);
+      socket.destroy();
     }
   });
 }
@@ -74,50 +83,45 @@ async function handleDeviceSocketRequest(
   socket: Socket,
   head: Buffer,
 ) {
-  try {
-    // Authenticate device
-    const device = await authenticateDeviceRequest(req);
-    if (!device) {
-      return socket.destroy();
-    }
+  // Authenticate device
+  const device = await authenticateDeviceRequest(req);
+  if (!device) {
+    return socket.destroy();
+  }
 
-    // Inflight means that the device has connected, a client has connected to that device via HTTP, and they're now doing the signaling dance
-    if (inFlight.has(device.id)) {
-      console.log(
-        `[Device] Device ${device.id} already has an inflight client connection.`,
-      );
-      return socket.destroy();
-    }
+  // Inflight means that the device has connected, a client has connected to that device via HTTP, and they're now doing the signaling dance
+  if (inFlight.has(device.id)) {
+    console.log(
+      `[Device] Device ${device.id} already has an inflight client connection.`,
+    );
+    return socket.destroy();
+  }
 
-    // Handle existing connections for this device
-    if (activeConnections.has(device.id)) {
-      console.log(
-        `[Device] Device ${device.id} already connected. Terminating existing connection.`,
-      );
-
-      const existingDeviceWs = activeConnections.get(device.id)!.ws;
-      await new Promise(resolve => {
-        console.log("[Device] Waiting for existing connection to close...");
-        existingDeviceWs.on("close", () => {
-          activeConnections.delete(device.id);
-          console.log("[Device] Existing connection closed.");
-
-          // Now we continue with the new connection
-          resolve(true);
-        });
+  // Handle existing connections for this device
+  if (activeConnections.has(device.id)) {
+    console.log(
+      `[Device] Device ${device.id} already connected. Terminating existing connection.`,
+    );
+
+    const existingDeviceWs = activeConnections.get(device.id)!.ws;
+    await new Promise(resolve => {
+      console.log("[Device] Waiting for existing connection to close...");
+      existingDeviceWs.on("close", () => {
+        activeConnections.delete(device.id);
+        console.log("[Device] Existing connection closed.");
 
-        existingDeviceWs.terminate();
+        // Now we continue with the new connection
+        resolve(true);
       });
-    }
 
-    // Complete the WebSocket upgrade
-    wssDevice.handleUpgrade(req, socket, head, ws => {
-      setupDeviceWebSocket(ws, device, req);
+      existingDeviceWs.terminate();
     });
-  } catch (error) {
-    console.error("Error handling device socket request:", error);
-    socket.destroy();
   }
+
+  // Complete the WebSocket upgrade
+  wssDevice.handleUpgrade(req, socket, head, ws => {
+    setupDeviceWebSocket(ws, device, req);
+  });
 }
 
 // Authenticate the device connection
@@ -190,10 +194,9 @@ function setupDeviceWebSocket(deviceWs: WebSocket, device: Device, req: Incoming
     // We check for aliveness every 10s
   }, 10000);
 
-  // Handle errors and connection close
-  deviceWs.on("error", async error => {
+  // Handle errors and connection close. ws emits "close" after "error", so cleanup runs once, there.
+  deviceWs.on("error", error => {
     console.log(`[Device] Error for ${id}:`, error);
-    await cleanup();
   });
 
   deviceWs.on("close", async (code, reason) => {
diff --git a/test/releases.test.ts b/test/releases.test.ts
index 7d18033..dd16207 100644
--- a/test/releases.test.ts
+++ b/test/releases.test.ts
@@ -80,7 +80,8 @@ function mockS3HashFile(
     Contents: [],
   });
 
-  // Mock legacy hash path
+  // The legacy artifact and its hash
+  s3Mock.on(HeadObjectCommand, { Key: artifactPath }).resolves({});
   s3Mock.on(GetObjectCommand, { Key: `${artifactPath}.sha256` }).resolves({
     Body: createAsyncIterable(hash) as any,
   });
@@ -324,6 +325,34 @@ describe("Retrieve handler", () => {
   });
 
   describe("prerelease mode", () => {
+    it("checks only the five newest versions for a SKU that none of them ships", async () => {
+      // Defaults first: a later mock for the same command takes precedence.
+      s3Mock.on(ListObjectsV2Command, { MaxKeys: 1 }).resolves({ Contents: [{ Key: "x" }] });
+      s3Mock
+        .on(HeadObjectCommand)
+        .rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
+      // 20 app versions with a skus/ folder; only the oldest, 1.0.0, has an SDMMC build.
+      // The system part has a valid SDMMC build, so only the app part fails.
+      mockS3ListVersions("app", Array.from({ length: 20 }, (_, i) => `1.${i}.0`));
+      mockS3ListVersions("system", ["1.0.0"]);
+      mockS3SkuVersion("system", "1.0.0", "jetkvm-v2-sdmmc", "system-hash");
+      s3Mock
+        .on(HeadObjectCommand, { Key: "app/1.0.0/skus/jetkvm-v2-sdmmc/jetkvm_app" })
+        .resolves({});
+
+      await expect(
+        Retrieve(
+          createMockRequest({ deviceId: "device-123", prerelease: "true", sku: "jetkvm-v2-sdmmc" }),
+          createMockResponse(),
+        ),
+      ).rejects.toThrow(NotFoundError);
+      // One app version listing, then one skus/ check per version for the newest five
+      const appListings = s3Mock
+        .commandCalls(ListObjectsV2Command)
+        .filter(call => call.args[0].input.Prefix?.startsWith("app/"));
+      expect(appListings).toHaveLength(1 + 5);
+    });
+
     it("should return latest prerelease version when prerelease=true", async () => {
       const req = createMockRequest({ deviceId: "device-123", prerelease: "true" });
       const res = createMockResponse();
@@ -455,6 +484,16 @@ describe("Retrieve handler", () => {
       });
     });
 
+    it("pins only the requested part; the other follows the rollout", async () => {
+      await createDbReleasePair("3.0.0", 100);
+      await createDbReleasePair("3.1.0", 0);
+
+      const res = createMockResponse();
+      await Retrieve(createMockRequest({ deviceId: "pinned-device", appVersion: "3.1.0" }), res);
+
+      expect(jsonBody(res)).toMatchObject({ appVersion: "3.1.0", systemVersion: "3.0.0" });
+    });
+
     it("omits DB-backed stable sigUrl fields when sibling .sig objects are absent", async () => {
       await createDbReleasePair("3.1.1", 100);
       mockArtifactSig("app", "3.1.1");
@@ -913,6 +952,20 @@ describe("RetrieveLatestApp S3 redirect handler", () => {
     clearCaches();
   });
 
+  it("answers 404, without an older build, when the newest version has no build for the SKU", async () => {
+    // 1.1.0 ships only JetKVM v2; 1.0.0 also has an SDMMC build
+    mockS3ListVersions("app", ["1.0.0", "1.1.0"]);
+    s3Mock.on(ListObjectsV2Command, { MaxKeys: 1 }).resolves({ Contents: [{ Key: "x" }] });
+    s3Mock.on(HeadObjectCommand).rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
+    s3Mock
+      .on(HeadObjectCommand, { Key: "app/1.0.0/skus/jetkvm-v2-sdmmc/jetkvm_app" })
+      .resolves({});
+
+    await expect(
+      RetrieveLatestApp(createMockRequest({ sku: "jetkvm-v2-sdmmc" }), createMockResponse()),
+    ).rejects.toThrow(NotFoundError);
+  });
+
   it("should handle all invalid semver versions gracefully", async () => {
     const req = createMockRequest({});
     const res = createMockResponse();
@@ -1259,6 +1312,23 @@ describe("RetrieveLatestSystemRecovery S3 redirect handler", () => {
     clearCaches();
   });
 
+  it("answers 404, without an older image, when the newest version has no image for the SKU", async () => {
+    // 1.1.0 ships only JetKVM v2; 1.0.0 also has an SDMMC image
+    mockS3ListVersions("system", ["1.0.0", "1.1.0"]);
+    s3Mock.on(ListObjectsV2Command, { MaxKeys: 1 }).resolves({ Contents: [{ Key: "x" }] });
+    s3Mock.on(HeadObjectCommand).rejects({ name: "NotFound", $metadata: { httpStatusCode: 404 } });
+    s3Mock
+      .on(HeadObjectCommand, { Key: "system/1.0.0/skus/jetkvm-v2-sdmmc/update_sd.img.zip" })
+      .resolves({});
+
+    await expect(
+      RetrieveLatestSystemRecovery(
+        createMockRequest({ sku: "jetkvm-v2-sdmmc" }),
+        createMockResponse(),
+      ),
+    ).rejects.toThrow(NotFoundError);
+  });
+
   it("should handle all invalid semver versions gracefully", async () => {
     const req = createMockRequest({});
     const res = createMockResponse();
diff --git a/vitest.e2e.config.ts b/vitest.e2e.config.ts
new file mode 100644
index 0000000..2bed800
--- /dev/null
+++ b/vitest.e2e.config.ts
@@ -0,0 +1,13 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    environment: "node",
+    include: ["e2e/**/*.test.ts"],
+    globalSetup: ["./e2e/global-setup.ts"],
+    // One API process and one database for the whole run
+    fileParallelism: false,
+    testTimeout: 20000,
+    hookTimeout: 60000,
+  },
+});