fix(server): reduce dropped frames in agent preview recordings

apps/server/src/preview/ServerBrowser.test.ts

@@ -260,27 +260,60 @@ function makeEncoderPage() {
     width: 0,
     height: 0,
     getContext: () => ({ drawImage() {}, fillRect() {} }),
-    captureStream: (frameRate?: number) => ({ frameRate, getTracks: () => [] }),
+    captureStream: (frameRate?: number) => ({
+      frameRate,
+      getTracks: () => [],
+      getVideoTracks: () => [{ requestFrame() {} }],
+    }),
+  };
+  const video = {
+    play: async () => {},
+    pause() {},
+    requestVideoFrameCallback: (callback: () => void) => queueMicrotask(callback),
   };
   class FakeMediaRecorder {
     static isTypeSupported = (type: string) => type === "video/webm";
     onstart: (() => void) | null = null;
+    onstop: (() => void) | null = null;
+    ondataavailable: ((event: { data: Blob }) => void) | null = null;
+    state = "inactive";
+    mimeType = "video/webm";
+    readonly stream: { frameRate?: number };
     constructor(stream: { frameRate?: number }, options: { videoBitsPerSecond: number }) {
+      this.stream = stream;
       encoderStreams.push({
         frameRate: stream.frameRate,
         bitsPerSecond: options.videoBitsPerSecond,
       });
     }
     start() {
+      this.state = "recording";
       this.onstart?.();
     }
+    stop() {
+      this.state = "inactive";
+      encodedAtStop = [...decoded];
+      this.ondataavailable?.({ data: new Blob(["video"]) });
+      this.onstop?.();
+    }
   }
   const sandbox: Record<string, unknown> = {
-    document: { createElement: () => canvas },
+    document: { createElement: (tag: string) => (tag === "video" ? video : canvas) },
     Path2D: function Path2D() {},
     MediaRecorder: FakeMediaRecorder,
-    createImageBitmap: async () => ({ width: 1280, height: 800, close() {} }),
-    atob,
+    createImageBitmap: async (blob: Blob) => {
+      const decoding = blob.text();
+      await decodeGate?.promise;
+      decoded.push(await decoding);
+      if (decoded.length === decodedTarget?.count) decodedTarget.reached.resolve();
+      return { width: 1280, height: 800, close() {} };
+    },
+    Uint8Array: class extends Uint8Array {
+      // The pinned Chrome has Uint8Array.fromBase64; Node does not without a flag.
+      static override fromBase64 = (base64: string) =>
+        new Uint8Array(Buffer.from(base64, "base64"));
+    },
+    btoa,
     Blob,
     performance,
   };
@@ -300,6 +333,10 @@ const contexts: ReturnType<typeof makeContext>[] = [];
 const encoderStreams: Array<{ frameRate: number | undefined; bitsPerSecond: number }> = [];
 const contextRequests: Array<{ profileId: string; isolated: boolean }> = [];
 let contextGate: PromiseWithResolvers<void> | null = null;
+let decodeGate: PromiseWithResolvers<void> | null = null;
+const decoded: Array<string> = [];
+let decodedTarget: { count: number; reached: PromiseWithResolvers<void> } | null = null;
+let encodedAtStop: Array<string> | null = null;
 let pageCopy = "";
 let keyCopy = "";
 let focusedFrame: ReturnType<typeof makeFrame> | null = null;
@@ -477,6 +514,10 @@ beforeEach(() => {
   encoderStreams.length = 0;
   contextRequests.length = 0;
   contextGate = null;
+  decodeGate = null;
+  decoded.length = 0;
+  decodedTarget = null;
+  encodedAtStop = null;
   contextFailure = null;
   pageSetupFailure = null;
   initialNavigation = null;
@@ -818,6 +859,71 @@ it.live.each([
     ).pipe(Effect.provide(layer)),
 );
 
+// Chrome stops sending screencast frames while too many are unacked.
+it.live("recordings keep receiving screencast frames while the encoder decodes", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { broker, tabId } = yield* ready;
+      yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
+      const session = contexts[0]!.sessions.find((candidate) =>
+        candidate.send.mock.calls.some(([method]) => method === "Page.startScreencast"),
+      )!;
+      decodeGate = Promise.withResolvers<void>();
+      decodedTarget = { count: decoded.length + 2, reached: Promise.withResolvers<void>() };
+      for (const [sessionId, data] of [
+        [1, "first"],
+        [2, "stale"],
+        [3, "latest"],
+      ] as const) {
+        session.emit("Page.screencastFrame", {
+          data: Buffer.from(data).toString("base64"),
+          sessionId,
+          metadata: { deviceWidth: 1280 },
+        });
+      }
+      expect(
+        session.send.mock.calls.filter(([method]) => method === "Page.screencastFrameAck"),
+      ).toHaveLength(3);
+      decodeGate.resolve();
+      yield* Effect.promise(() => decodedTarget!.reached.promise);
+      expect(decoded.slice(-2)).toEqual(["first", "latest"]);
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
+it.live("stopping a recording encodes the newest frame still waiting on the encoder", () =>
+  Effect.scoped(
+    Effect.gen(function* () {
+      const { broker, tabId } = yield* ready;
+      yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
+      const session = contexts[0]!.sessions.find((candidate) =>
+        candidate.send.mock.calls.some(([method]) => method === "Page.startScreencast"),
+      )!;
+      const gate = Promise.withResolvers<void>();
+      decodeGate = gate;
+      const send = session.send.getMockImplementation()!;
+      session.send.mockImplementation(async (method, input) => {
+        // A stop that skips the encoder finishes on microtasks, before this runs.
+        if (method === "Page.stopScreencast") setImmediate(gate.resolve);
+        return send(method, input);
+      });
+      for (const [sessionId, data] of [
+        [1, "decoding"],
+        [2, "stale"],
+        [3, "latest"],
+      ] as const) {
+        session.emit("Page.screencastFrame", {
+          data: Buffer.from(data).toString("base64"),
+          sessionId,
+          metadata: { deviceWidth: 1280 },
+        });
+      }
+      yield* broker.invoke({ scope, tabId, operation: "recordingStop", input: {} });
+      expect(encodedAtStop?.slice(-2)).toEqual(["decoding", "latest"]);
+    }),
+  ).pipe(Effect.provide(layer)),
+);
+
 it.live("enforces provider ownership and explicit targets when a session has multiple tabs", () =>
   Effect.scoped(
     Effect.gen(function* () {

apps/server/src/preview/ServerBrowser.ts

@@ -286,7 +286,7 @@ interface Recording {
   readonly encoder: Page;
   readonly session: CDPSession;
   readonly startedAt: string;
-  readonly framesInFlight: Set<Promise<void>>;
+  readonly drained: () => Promise<void>;
 }
 
 interface ServerTab {
@@ -1744,34 +1744,42 @@ const make = Effect.gen(function* () {
         );
         const opened = await tab.page.context().newCDPSession(tab.page);
         session = opened;
-        const framesInFlight = new Set<Promise<void>>();
+        // Acks never wait on the encoder, or Chrome throttles the screencast to its
+        // decode latency. A decode in progress keeps only the newest frame waiting.
+        let pending: { readonly data: string; readonly cssWidth: number } | null = null;
+        let delivering: Promise<void> | null = null;
+        const deliver = async () => {
+          while (pending) {
+            const { data, cssWidth } = pending;
+            pending = null;
+            const accepted = await encoder
+              .evaluate(
+                ([data, width]) =>
+                  (globalThis as unknown as EncoderWindow).__t3Recorder.frame(data, width),
+                [data, cssWidth] as const,
+              )
+              .catch(() => true);
+            if (!accepted) await opened.send("Page.stopScreencast").catch(constVoid);
+          }
+          delivering = null;
+        };
         opened.on("Page.screencastFrame", (frame) => {
-          const cssWidth = tab.page.viewportSize()?.width ?? frame.metadata.deviceWidth;
-          const delivered: Promise<void> = encoder
-            .evaluate(
-              ([data, width]) =>
-                (globalThis as unknown as EncoderWindow).__t3Recorder.frame(data, width),
-              [frame.data, cssWidth] as const,
-            )
-            .then(async (accepted) => {
-              if (!accepted) await opened.send("Page.stopScreencast");
-            })
-            .catch(constVoid)
-            .finally(() => {
-              framesInFlight.delete(delivered);
-              void opened
-                .send("Page.screencastFrameAck", { sessionId: frame.sessionId })
-                .catch(constVoid);
-            });
-          framesInFlight.add(delivered);
+          void opened
+            .send("Page.screencastFrameAck", { sessionId: frame.sessionId })
+            .catch(constVoid);
+          pending = {
+            data: frame.data,
+            cssWidth: tab.page.viewportSize()?.width ?? frame.metadata.deviceWidth,
+          };
+          delivering ??= deliver();
         });
         await opened.send("Page.startScreencast", RECORDING_SCREENCAST);
         if (tab.closing) throw new Error("The tab closed while the recording started.");
         const recording: Recording = {
           encoder,
           session: opened,
           startedAt: new Date().toISOString(),
-          framesInFlight,
+          drained: () => delivering ?? Promise.resolve(),
         };
         tab.recording = recording;
         return recording;
@@ -1822,7 +1830,7 @@ const make = Effect.gen(function* () {
       const chunks: Array<Buffer> = [];
       try {
         await recording.session.send("Page.stopScreencast").catch(constVoid);
-        await Promise.all(recording.framesInFlight);
+        await recording.drained();
         await recording.session.detach().catch(constVoid);
         const stopped = await recording.encoder.evaluate(() =>
           (globalThis as unknown as EncoderWindow).__t3Recorder.stop(),

apps/server/src/preview/ServerBrowserPage.ts

@@ -1056,8 +1056,7 @@ export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) =>
   window.__t3Recorder = {
     async frame(base64, cssWidth) {
       if (tooLarge) return false;
-      const bytes = Uint8Array.from(atob(base64), (char) => char.charCodeAt(0));
-      const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" }));
+      const bitmap = await createImageBitmap(new Blob([Uint8Array.fromBase64(base64)], { type: "image/jpeg" }));
       if (tooLarge) { bitmap.close(); return false; }
       if (!recorder) {
         canvas.width = bitmap.width;