diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 6db04b389ab6d..6e823fed3c678 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -148,6 +148,7 @@ const { createFromPath, fromId, getFocusedWebContents, + getDisplayMatching, mkdir, previewSession, showItemInFolder, @@ -163,6 +164,7 @@ const { })), fromId: vi.fn<(_id?: number) => Electron.WebContents | null>((_id?: number) => null), getFocusedWebContents: vi.fn(() => null), + getDisplayMatching: vi.fn(() => ({ scaleFactor: 1 })), mkdir: vi.fn((_path: string) => undefined), previewSession: { on: vi.fn() }, showItemInFolder: vi.fn(), @@ -186,6 +188,7 @@ vi.mock("electron", () => ({ nativeImage: { createFromPath, }, + screen: { getDisplayMatching }, shell: { showItemInFolder, }, @@ -271,6 +274,7 @@ const withManager = ( interface TestCapturedPreviewImage { readonly toJPEG: () => Buffer; readonly toBitmap?: () => Buffer; + readonly resize?: (size: { width: number; height: number }) => TestCapturedPreviewImage; readonly getSize: () => { readonly width: number; readonly height: number }; } @@ -524,6 +528,7 @@ const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () const webContentsListeners = new Map void>(); const send = vi.fn(); let destroyed = false; + let contentSize = [480, 320]; const webContents = { on: vi.fn((event: string, listener: () => void) => { webContentsListeners.set(event, listener); @@ -541,8 +546,11 @@ const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () setAlwaysOnTop: vi.fn(), setVisibleOnAllWorkspaces: vi.fn(), setAspectRatio: vi.fn(), - getContentSize: vi.fn(() => [480, 320]), - setContentSize: vi.fn(), + getBounds: vi.fn(() => ({ x: 0, y: 0, width: contentSize[0], height: contentSize[1] })), + getContentSize: vi.fn(() => contentSize), + setContentSize: vi.fn((width: number, height: number) => { + contentSize = [width, height]; + }), loadURL: vi.fn(loadURL), showInactive: vi.fn(() => { if (destroyed) throw new Error("Picture-in-picture window is closed."); @@ -563,6 +571,8 @@ const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () describe("PreviewManager", () => { beforeEach(() => { browserWindowConstructor.mockReset(); + getDisplayMatching.mockReset(); + getDisplayMatching.mockReturnValue({ scaleFactor: 1 }); fromId.mockClear(); getFocusedWebContents.mockReset(); getFocusedWebContents.mockReturnValue(null); @@ -2638,10 +2648,77 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "sizes PiP pixels for its display and resize without changing source layout", + () => + withManager((manager) => + Effect.gen(function* () { + const frames = yield* Queue.unbounded<{ width: number; height: number; data: string }>(); + let sourceSize = { width: 1280, height: 720 }; + const encodedSizes: Array<{ width: number; height: number }> = []; + const resizeTargets: Array<{ width: number; height: number }> = []; + const imageAt = (size: { width: number; height: number }): TestCapturedPreviewImage => ({ + getSize: () => size, + toBitmap: () => Buffer.from("unchanged page"), + toJPEG: () => { + encodedSizes.push(size); + return Buffer.from(JSON.stringify(size)); + }, + resize: ({ width, height }) => { + resizeTargets.push({ width, height }); + return imageAt({ width, height }); + }, + }); + fromId.mockReturnValue(makeTestPreviewWebContents(async () => imageAt(sourceSize))); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + send.mockImplementation((_channel, frame) => { + Queue.offerUnsafe(frames, frame); + }); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + yield* manager.createTab("tab_pip_pixels"); + yield* manager.registerWebview("tab_pip_pixels", 42); + yield* manager.openPictureInPicture("tab_pip_pixels"); + const first = yield* Queue.take(frames); + expect(first).toMatchObject(sourceSize); + expect(encodedSizes).toEqual([{ width: 523, height: 294 }]); + expect(resizeTargets).toEqual([{ width: 523, height: 294 }]); + + yield* TestClock.adjust(84); + expect(encodedSizes).toHaveLength(1); + expect(resizeTargets).toHaveLength(1); + for (const [width, height, scaleFactor, expected] of [ + [400, 225, 1, { width: 400, height: 225 }], + [400, 225, 2, { width: 800, height: 450 }], + [400, 225, 1.25, { width: 500, height: 281 }], + [1600, 900, 2, { width: 1280, height: 720 }], + ] as const) { + pictureInPictureWindow.setContentSize(width, height); + getDisplayMatching.mockReturnValue({ scaleFactor }); + yield* TestClock.adjust(84); + const frame = yield* Queue.take(frames); + expect(frame).toMatchObject(sourceSize); + expect(JSON.parse(Buffer.from(frame.data, "base64").toString())).toEqual(expected); + } + expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]); + + sourceSize = { width: 720, height: 1280 }; + pictureInPictureWindow.setContentSize(225, 400); + getDisplayMatching.mockReturnValue({ scaleFactor: 1 }); + yield* TestClock.adjust(84); + expect(yield* Queue.take(frames)).toMatchObject(sourceSize); + expect(encodedSizes.at(-1)).toEqual({ width: 240, height: 427 }); + expect(pictureInPictureWindow.setAspectRatio.mock.calls.at(-1)).toEqual([720 / 1280]); + }), + ), + ); + effectIt.effect("shows picture-in-picture and keeps capturing after stalled captures", () => withManager((manager) => Effect.gen(function* () { const image = { + resize: () => image, toJPEG: () => Buffer.from("preview-frame"), getSize: () => ({ width: 1280, height: 720 }), }; diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 189885b9b2178..2616959b86096 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -36,6 +36,7 @@ import { type Session, clipboard, nativeImage, + screen, shell, webContents, } from "electron"; @@ -2692,35 +2693,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } const frameSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); if (frameSession?.scope !== captureSession.scope) return; - // Fingerprint before encoding so unchanged frames skip JPEG, base64, and IPC; unreadable frames must still be sent. - const fingerprint = fingerprintCapturedFrame(image, size); - const pictureInPicture = - frameSession.consumers.has("picture-in-picture") && - (fingerprint === null || frameSession.lastPictureInPictureFingerprint !== fingerprint); - if (!pictureInPicture) return; - const encoded = yield* attempt( - { - operation: "frameCapture.encodeFrame", - tabId, - webContentsId: wc.id, - }, - () => image.toJPEG(PICTURE_IN_PICTURE_JPEG_QUALITY), - ); - const receivedAt = yield* currentIso; - const frame: DesktopPreviewRecordingFrame = { - tabId, - data: encoded.toString("base64"), - width: size.width, - height: size.height, - receivedAt, - }; const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, )?.window; if (!pictureInPictureWindow || pictureInPictureWindow.isDestroyed()) return; yield* Effect.gen(function* () { const previousAspectRatio = (yield* Ref.get(pictureInPictureAspectRatiosRef)).get(tabId); - const aspectRatio = frame.width / frame.height; + const aspectRatio = size.width / size.height; if ( previousAspectRatio === undefined || Math.abs(previousAspectRatio - aspectRatio) > PICTURE_IN_PICTURE_ASPECT_RATIO_EPSILON @@ -2747,6 +2726,55 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); } + const previewSize = yield* attempt( + { operation: "frameCapture.measureDestination", tabId, webContentsId: wc.id }, + () => { + const [width, height] = pictureInPictureWindow.getContentSize(); + const { scaleFactor } = screen.getDisplayMatching(pictureInPictureWindow.getBounds()); + const scale = Math.min( + 1, + Math.ceil(width! * scaleFactor) / size.width, + Math.ceil(height! * scaleFactor) / size.height, + ); + return { + width: Math.max(1, Math.round(size.width * scale)), + height: Math.max(1, Math.round(size.height * scale)), + }; + }, + ); + // Fingerprint before encoding so unchanged frames skip JPEG, base64, and IPC; unreadable frames must still be sent. + const sourceFingerprint = fingerprintCapturedFrame(image, size); + const fingerprint = + sourceFingerprint === null + ? null + : `${sourceFingerprint}:${previewSize.width}x${previewSize.height}`; + const pictureInPicture = + frameSession.consumers.has("picture-in-picture") && + (fingerprint === null || frameSession.lastPictureInPictureFingerprint !== fingerprint); + if (!pictureInPicture) return; + const encoded = yield* attempt( + { + operation: "frameCapture.encodeFrame", + tabId, + webContentsId: wc.id, + }, + () => { + const previewImage = + previewSize.width !== size.width || previewSize.height !== size.height + ? image.resize(previewSize) + : image; + return previewImage.toJPEG(PICTURE_IN_PICTURE_JPEG_QUALITY); + }, + ); + const receivedAt = yield* currentIso; + // Keep source dimensions for layout; JPEG pixels only serve the PiP display. + const frame: DesktopPreviewRecordingFrame = { + tabId, + data: encoded.toString("base64"), + width: size.width, + height: size.height, + receivedAt, + }; yield* attempt( { operation: "pictureInPicture.deliverFrame",