Release dev → main: 0% default rollout, staged releases for new SKUs, compiled image, e2e tests

.dockerignore

@@ -5,3 +5,5 @@ dist/
 .DS_Store
 .env
 .git
+.env.*
+!.env.example

.github/workflows/pull-request.yml

@@ -28,10 +28,14 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    # Pulls and starts in the background while dependencies install; the end-to-end tests use it
+    - name: Start S3-compatible store
+      run: nohup docker compose -f compose.development.yaml up -d s3 > "$RUNNER_TEMP/s3-start.log" 2>&1 &
+
     - name: Use Node.js
       uses: actions/setup-node@v4
       with:
-        node-version: v22.21.0
+        node-version: v22.23.4
         cache: 'npm'
         cache-dependency-path: '**/package-lock.json'
 
@@ -57,6 +61,11 @@ jobs:
     - name: Run Tests
       run: npm test
 
+    - name: Run end-to-end tests
+      run: |
+        timeout 180 sh -c 'until curl -s -o /dev/null http://127.0.0.1:8333; do sleep 1; done' || { cat "$RUNNER_TEMP/s3-start.log"; exit 1; }
+        npx vitest run --config vitest.e2e.config.ts
+
     - name: Cache Prisma Binary
       uses: actions/cache@v4
       with:

Dockerfile

@@ -1,21 +1,34 @@
-FROM node:22.23.4-bookworm-slim
+FROM node:22.23.4-bookworm-slim AS base
 LABEL org.opencontainers.image.source="https://github.com/jetkvm/cloud-api"
 WORKDIR /usr/src/app
 
-# Prisma needs OpenSSL at runtime
+# Prisma needs OpenSSL at runtime; tini forwards SIGTERM to node as PID 1
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends openssl ca-certificates \
+  && apt-get install -y --no-install-recommends openssl ca-certificates tini \
   && rm -rf /var/lib/apt/lists/*
 
+FROM base AS build
 COPY package.json package-lock.json ./
 COPY prisma ./prisma
-# ts-node type-checks at startup, so the @types/* devDependencies are needed at runtime
 RUN npm ci
+COPY tsconfig.json ./
+COPY src ./src
+# Type errors fail the build here instead of crashing the container at startup
+RUN npx prisma generate \
+  && npm run build \
+  && npm prune --omit=dev
 
-COPY . .
-RUN npx prisma generate
-
+FROM base
 ENV NODE_ENV=production
 ENV PORT=3000
+COPY package.json ./
+COPY prisma ./prisma
+COPY --from=build /usr/src/app/node_modules ./node_modules
+COPY --from=build /usr/src/app/dist ./dist
+
+USER node
 EXPOSE 3000
-CMD ["npm", "run", "start"]
+HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
+  CMD ["node", "-e", "fetch('http://localhost:' + process.env.PORT + '/healthz').then(r => process.exit(r.ok ? 0 : 1), () => process.exit(1))"]
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["node", "dist/index.js"]

README.md

@@ -28,7 +28,7 @@ If you've found an issue and want to report it, please check our [Issues](https:
 This project is built with Node.js, Prisma, and Express.
 
 ```bash
-# Start the database
+# Start the database and the S3-compatible store the e2e tests use
 docker compose -f compose.development.yaml up -d
 
 # Copy and configure environment variables
@@ -50,6 +50,29 @@ npm run dev
 npm test
 ```
 
+### End-to-end tests
+
+`npm run test:e2e` builds the API and starts it as the container does, against a
+database (`jetkvm_e2e`) and a bucket that only the e2e run uses. It needs the
+services from `compose.development.yaml`. Override them with `E2E_DATABASE_URL`
+and `E2E_S3_ENDPOINT`.
+
+### Comparing two deployments
+
+`npm run compare` sends the same requests to two deployments of the API and
+reports where the answers differ, grouped by cause. In a terminal it asks you to
+mark each group as expected or not, and exits with 0 only when you mark them all:
+
+```bash
+npm run compare -- --candidate local
+npm run compare -- --candidate staging --only releases,redirects
+npm run compare -- --help
+```
+
+The reference defaults to `https://api.jetkvm.com`. Update checks that depend
+on the rollout table differ unless both databases have the same rollout
+percentages.
+
 ## Self-Hosting
 
 For self-hosting, use the default compose file which runs the complete stack:

compose.development.yaml

@@ -19,6 +19,14 @@ services:
     volumes:
       - postgresql:/var/lib/postgresql
 
+  # S3-compatible store for the e2e tests (npm run test:e2e)
+  s3:
+    image: chrislusf/seaweedfs:4.50
+    restart: unless-stopped
+    command: server -s3 -dir=/data
+    ports:
+      - "8333:8333"
+
 volumes:
   postgresql:
     driver: local

e2e/devices.test.ts

@@ -0,0 +1,125 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import {
+  api,
+  closed,
+  createUserWithDevice,
+  db,
+  openSocket,
+  sessionCookie,
+  unsignedIdToken,
+} from "./helpers";
+
+beforeEach(async () => {
+  await db.device.deleteMany({ where: { id: { startsWith: "e2e-dev-" } } });
+  await db.user.deleteMany({ where: { googleId: { startsWith: "e2e-dev-" } } });
+});
+
+function exchange(tempToken?: string) {
+  return api("/devices/token", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(tempToken === undefined ? {} : { tempToken }),
+  });
+}
+
+async function deviceWithTempToken(id: string, tempToken: string, expiresInMs: number) {
+  await db.user.create({
+    data: {
+      googleId: `e2e-dev-${id}-owner`,
+      device: {
+        create: { id, tempToken, tempTokenExpiresAt: new Date(Date.now() + expiresInMs) },
+      },
+    },
+  });
+}
+
+describe("device registration (POST /devices/token)", () => {
+  it("trades a temporary token for a secret token the device can connect with, once", async () => {
+    await deviceWithTempToken("e2e-dev-1", "temp-1", 60_000);
+
+    const response = await exchange("temp-1");
+    expect(response.status).toBe(200);
+    const { secretToken } = await response.json();
+    expect(secretToken).toMatch(/^[0-9a-f]{40}$/);
+
+    const device = await db.device.findUniqueOrThrow({ where: { id: "e2e-dev-1" } });
+    expect(device).toMatchObject({
+      secretToken,
+      tempToken: null,
+      tempTokenExpiresAt: null,
+    });
+
+    // The temporary token is single-use
+    expect((await exchange("temp-1")).status).toBe(404);
+
+    const ws = await openSocket("/", {
+      authorization: `Bearer ${secretToken}`,
+      "x-device-id": "e2e-dev-1",
+    });
+    ws.close();
+    await closed(ws);
+  });
+
+  it("refuses an expired temporary token and leaves the device without a secret", async () => {
+    await deviceWithTempToken("e2e-dev-2", "temp-2", -1_000);
+    expect((await exchange("temp-2")).status).toBe(401);
+    expect(
+      (await db.device.findUniqueOrThrow({ where: { id: "e2e-dev-2" } })).secretToken,
+    ).toBeNull();
+  });
+
+  it("answers 404 to an unknown token and 422 to a missing one", async () => {
+    expect((await exchange("never-issued")).status).toBe(404);
+    expect((await exchange()).status).toBe(422);
+  });
+});
+
+describe("device removal (DELETE /devices/:id)", () => {
+  it("lets the device remove itself with its secret token", async () => {
+    await createUserWithDevice("e2e-dev-owner-3", "e2e-dev-3", "secret-3");
+
+    const response = await api("/devices/e2e-dev-3", {
+      method: "DELETE",
+      headers: { authorization: "Bearer secret-3" },
+    });
+    expect(response.status).toBe(204);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-3" } })).toBeNull();
+  });
+
+  it("answers 404 to a secret token that belongs to no device", async () => {
+    await createUserWithDevice("e2e-dev-owner-4", "e2e-dev-4", "secret-4");
+    const response = await api("/devices/e2e-dev-4", {
+      method: "DELETE",
+      headers: { authorization: "Bearer someone-elses-secret" },
+    });
+    expect(response.status).toBe(404);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-4" } })).not.toBeNull();
+  });
+
+  it("answers 401 without credentials and keeps serving", async () => {
+    await createUserWithDevice("e2e-dev-owner-5", "e2e-dev-5", "secret-5");
+
+    const response = await api("/devices/e2e-dev-5", {
+      method: "DELETE",
+      signal: AbortSignal.timeout(5000),
+    });
+    expect(response.status).toBe(401);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-5" } })).not.toBeNull();
+    expect((await api("/healthz")).status).toBe(200);
+  });
+
+  it("answers 401 to a session whose ID token Google did not sign", async () => {
+    await createUserWithDevice("e2e-dev-owner-6", "e2e-dev-6", "secret-6");
+    const cookie = sessionCookie({
+      id_token: unsignedIdToken({ sub: "e2e-dev-owner-6" }),
+    });
+
+    const response = await api("/devices/e2e-dev-6", {
+      method: "DELETE",
+      headers: { cookie },
+      signal: AbortSignal.timeout(5000),
+    });
+    expect(response.status).toBe(401);
+    expect(await db.device.findUnique({ where: { id: "e2e-dev-6" } })).not.toBeNull();
+  });
+});

e2e/fixtures.ts

@@ -0,0 +1,75 @@
+/**
+ * The release bucket the e2e API reads. Global setup uploads every artifact
+ * with a `.sha256` sidecar (and a `.sig` where `sig` is set), as the release
+ * scripts do, then registers each stable version through POST /releases/sync.
+ */
+
+export const CDN_BASE = "https://cdn.e2e.test";
+
+interface FixtureArtifact {
+  key: string;
+  sig?: boolean;
+}
+
+export const ARTIFACTS: FixtureArtifact[] = [
+  // App. 0.3.0 predates the skus/ layout, 0.5.0 has no SDMMC build.
+  { key: "app/0.3.0/jetkvm_app" },
+  { key: "app/0.4.0/skus/jetkvm-v2/jetkvm_app", sig: true },
+  { key: "app/0.4.0/skus/jetkvm-v2-sdmmc/jetkvm_app" },
+  { key: "app/0.5.0/skus/jetkvm-v2/jetkvm_app" },
+  { key: "app/0.6.0-beta.1/skus/jetkvm-v2/jetkvm_app" },
+  { key: "app/0.6.0-beta.1/skus/jetkvm-v2-sdmmc/jetkvm_app" },
+
+  // System, with the recovery images that share its version.
+  { key: "system/0.2.0/system.tar" },
+  { key: "system/0.2.0/update.img" },
+  { key: "system/0.3.0/skus/jetkvm-v2/system.tar" },
+  { key: "system/0.3.0/skus/jetkvm-v2/update.img" },
+  { key: "system/0.3.0/skus/jetkvm-v2-sdmmc/system.tar" },
+  { key: "system/0.3.0/skus/jetkvm-v2-sdmmc/update_sd.img.zip" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2/system.tar" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2/update.img" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2-sdmmc/system.tar" },
+  { key: "system/0.4.0-rc.1/skus/jetkvm-v2-sdmmc/update_sd.img.zip" },
+
+  // Mini. 0.4.0 uses the pre-SKU layout, which no Mini build may use.
+  // 0.5.4 has no wireless build.
+  { key: "mini/0.4.0/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-ethernet/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-ethernet/jetkvm-mini-full.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini.bin" },
+  { key: "mini/0.5.3/skus/jetkvm-mini-wireless/jetkvm-mini-full.bin" },
+  { key: "mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini.bin" },
+  { key: "mini/0.5.4/skus/jetkvm-mini-ethernet/jetkvm-mini-full.bin" },
+];
+
+/** Objects that are not releases: a folder whose name is not a version. */
+export const OTHER_OBJECTS = ["app/latest/README.txt"];
+
+export function fixtureBody(key: string): Buffer {
+  return Buffer.from(`e2e fixture ${key}\n`);
+}
+
+/** Every (type, version) folder with no prerelease tag, as the upload script would sync them. */
+export function stableVersions(): { type: string; version: string }[] {
+  const seen = new Map<string, { type: string; version: string }>();
+  for (const { key } of ARTIFACTS) {
+    const [type, version] = key.split("/");
+    if (!version.includes("-")) seen.set(`${type}/${version}`, { type, version });
+  }
+  return [...seen.values()];
+}
+
+/**
+ * Device IDs and their rollout bucket: parseInt(md5(id).slice(0, 8), 16) % 100.
+ * The values are fixed so that a change to the bucketing, which would move
+ * devices between rollout groups mid-rollout, fails the tests.
+ */
+export const DEVICE_IN_BUCKET = {
+  0: "e2e-device-74",
+  9: "e2e-device-167",
+  49: "e2e-device-152",
+  50: "e2e-device-28",
+  81: "e2e-device-9",
+  99: "e2e-device-19",
+} as const;