diff --git a/.github/actions/ci-changes/action.yml b/.github/actions/ci-changes/action.yml index 7d2adc0c9e035..009d7ae0a9de5 100644 --- a/.github/actions/ci-changes/action.yml +++ b/.github/actions/ci-changes/action.yml @@ -21,6 +21,8 @@ outputs: value: ${{ steps.detect.outputs.release_smoke }} mobile_native: value: ${{ steps.detect.outputs.mobile_native }} + desktop_preview: + value: ${{ steps.detect.outputs.desktop_preview }} runs: using: composite steps: diff --git a/.github/scripts/ci-changes.cjs b/.github/scripts/ci-changes.cjs index 7b7332903ed9f..813f1e9586716 100644 --- a/.github/scripts/ci-changes.cjs +++ b/.github/scripts/ci-changes.cjs @@ -51,6 +51,11 @@ const BUILD_ONLY_DEPS = { t3: ["@t3tools/web"] }; const MOBILE_NATIVE = /^apps\/mobile\/.*\.(swift|kt|kts)$|^apps\/mobile\/(\.swiftlint\.yml|detekt\.yml|\.editorconfig|Brewfile)$|^scripts\/mobile-native-static-check\.ts$|^package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$|^\.github\/actions\/ci-changes\//; +// Real-Electron check that agent preview tabs paint with the app window hidden. +// Electron is pinned in the desktop manifest, so its upgrades rerun it too. +const DESKTOP_PREVIEW = + /^apps\/desktop\/src\/preview\/|^apps\/desktop\/scripts\/verify-background-preview|^apps\/desktop\/package\.json$|^\.github\/workflows\/ci\.yml$|^\.github\/scripts\/ci-changes\.cjs$|^\.github\/actions\/ci-changes\//; + // Startup, migration, settlement, and sidebar paths that decide whether an upgraded user's // threads show after launch. Build runs scripts/smoke-desktop-app.ts for them, so they do not // first fail in the release launch check. @@ -186,6 +191,7 @@ function decide(files, packages, full) { rust: everything || rust, release_smoke: scriptsChanged, mobile_native: files === null || files.some((file) => MOBILE_NATIVE.test(file)), + desktop_preview: files === null || files.some((file) => DESKTOP_PREVIEW.test(file)), }, }; } diff --git a/.github/scripts/ci-changes.d.cts b/.github/scripts/ci-changes.d.cts index 4640e89776b0d..ff7ebcdbbc95a 100644 --- a/.github/scripts/ci-changes.d.cts +++ b/.github/scripts/ci-changes.d.cts @@ -24,5 +24,6 @@ export function decide( rust: boolean; release_smoke: boolean; mobile_native: boolean; + desktop_preview: boolean; }; }; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bfb648fe368c..99e184abd1723 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: rust: ${{ steps.detect.outputs.rust }} release_smoke: ${{ steps.detect.outputs.release_smoke }} mobile_native: ${{ steps.detect.outputs.mobile_native }} + desktop_preview: ${{ steps.detect.outputs.desktop_preview }} steps: # Only the package manifests the script reads to build the dependency graph. # Depth 2 brings the base commit the change is diffed against. @@ -617,6 +618,38 @@ jobs: - name: Lint mobile native sources run: vp run lint:mobile + # Unit tests mock Electron, so only a real one with a real window manager can + # prove agent preview tabs still paint while the app window is hidden. + desktop_preview: + name: Desktop Background Preview + needs: changes + if: ${{ !cancelled() && needs.changes.outputs.desktop_preview != 'false' }} + runs-on: blacksmith-6vcpu-macos-27 + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + sparse-checkout: | + /* + !/.repos/ + sparse-checkout-cone-mode: false + + - name: Setup Vite+ + uses: voidzero-dev/setup-vp@250f29ce396baf5e8f24498e17c0dfdebabc26eb # v1.15.0 + with: + node-version-file: package.json + cache: true + run-install: | + args: + - --filter=@t3tools/desktop + + - name: Ensure Electron runtime is installed + run: vp run --filter @t3tools/desktop ensure:electron + + - name: Verify background preview tabs paint + run: node apps/desktop/scripts/verify-background-preview.mjs + release_smoke: permissions: contents: read @@ -662,6 +695,7 @@ jobs: test_server, rust, mobile_native_static_analysis, + desktop_preview, release_smoke, ] runs-on: blacksmith-2vcpu-ubuntu-2404 diff --git a/apps/desktop/scripts/verify-background-preview.electron.cjs b/apps/desktop/scripts/verify-background-preview.electron.cjs new file mode 100644 index 0000000000000..04e95d0ce6afe --- /dev/null +++ b/apps/desktop/scripts/verify-background-preview.electron.cjs @@ -0,0 +1,95 @@ +// Electron entry for verify-background-preview.mjs. CommonJS, because an ESM +// entry delays `ready` until its top-level await settles. +const NodeFS = require("node:fs"); +const NodeOS = require("node:os"); +const NodePath = require("node:path"); +const NodeURL = require("node:url"); +const { app, BrowserWindow, nativeImage } = require("electron"); + +const CAPTURE_TIMEOUT_MS = 5_000; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const withTimeout = (promise, ms) => + Promise.race([ + promise.then((value) => ({ ok: true, value })), + sleep(ms).then(() => ({ ok: false })), + ]); +const page = (html) => `data:text/html,${encodeURIComponent(html)}`; + +async function captureColor(tab, rgb) { + await tab.debugger.sendCommand("Runtime.evaluate", { + expression: `document.body.style.background = 'rgb(${rgb.join(",")})'`, + }); + const shot = await withTimeout( + tab.debugger.sendCommand("Page.captureScreenshot", { format: "png" }), + CAPTURE_TIMEOUT_MS, + ); + if (!shot.ok) return "timed out"; + const pixels = nativeImage.createFromBuffer(Buffer.from(shot.value.data, "base64")).toBitmap(); + const got = [pixels[2], pixels[1], pixels[0]]; + // Color management can shift a channel by a few levels. + return got.every((value, index) => Math.abs(value - rgb[index]) <= 6) + ? null + : `stale: rgb(${got.join(",")})`; +} + +async function main() { + const { backgroundTabWindowOptions } = await import( + NodeURL.pathToFileURL(NodePath.join(__dirname, "../src/preview/BackgroundTabWindow.ts")).href + ); + const preload = NodePath.join( + NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-background-preview-")), + "preload.cjs", + ); + NodeFS.writeFileSync(preload, ""); + // Each state closes its windows, which would otherwise quit the app. + app.on("window-all-closed", () => {}); + await app.whenReady(); + + const failures = []; + for (const state of ["hidden", "minimized"]) { + const appWindow = new BrowserWindow({ width: 480, height: 320, show: true }); + await appWindow.loadURL(page("T3 Code")); + await sleep(500); + if (state === "hidden") appWindow.hide(); + else appWindow.minimize(); + await sleep(1_500); + + // Created only after the app window is gone, as an agent's tab would be. + const tabWindow = new BrowserWindow( + backgroundTabWindowOptions({ + partition: `t3-background-preview-${state}`, + preload, + passkeyBridgeArgument: undefined, + }), + ); + await tabWindow.loadURL(page("")); + const tab = tabWindow.webContents; + tab.debugger.attach("1.3"); + for (const [step, rgb] of [ + [1, [0, 200, 0]], + [2, [200, 200, 0]], + [3, [0, 200, 200]], + ]) { + const failure = await captureColor(tab, rgb); + if (failure) failures.push(`app window ${state}, capture ${step}: ${failure}`); + await sleep(1_000); + } + tabWindow.destroy(); + appWindow.destroy(); + } + + if (failures.length > 0) { + console.error("Background preview tabs stopped painting:"); + for (const failure of failures) console.error(` - ${failure}`); + app.exit(1); + return; + } + console.log("Background preview tabs paint while the app window is hidden or minimized."); + NodeFS.writeFileSync(process.env.T3_BACKGROUND_PREVIEW_DONE, "ok"); + app.exit(0); +} + +main().catch((error) => { + console.error(error); + app.exit(1); +}); diff --git a/apps/desktop/scripts/verify-background-preview.mjs b/apps/desktop/scripts/verify-background-preview.mjs new file mode 100644 index 0000000000000..4b332b48a374a --- /dev/null +++ b/apps/desktop/scripts/verify-background-preview.mjs @@ -0,0 +1,38 @@ +// Proves agent preview tabs still paint while the app window is hidden or minimized, +// using the window options the preview manager ships. Unit tests mock Electron. +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodeModule from "node:module"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +const RUN_TIMEOUT_MS = 90_000; +const entry = NodePath.join( + NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), + "verify-background-preview.electron.cjs", +); +// The plain binary: the development launcher's renamed app bundle refuses data: URLs. +const electronPath = NodeModule.createRequire(import.meta.url)("electron"); +const { ELECTRON_RUN_AS_NODE: _ignored, ...env } = process.env; +// Only a run that finished every capture writes this, so an early clean quit still fails. +const done = NodePath.join( + NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-background-preview-")), + "done", +); +const child = NodeChildProcess.spawn(electronPath, [entry], { + stdio: "inherit", + env: { ...env, T3_BACKGROUND_PREVIEW_DONE: done }, +}); +const timer = setTimeout(() => { + console.error(`Background preview check did not finish within ${RUN_TIMEOUT_MS}ms.`); + child.kill("SIGKILL"); +}, RUN_TIMEOUT_MS); +child.on("exit", (code, signal) => { + clearTimeout(timer); + if (code === 0 && !NodeFS.existsSync(done)) { + console.error("Background preview check quit before it finished."); + process.exit(1); + } + process.exit(code ?? (signal ? 1 : 0)); +}); diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 57d7101cf9ec7..b19727a53b085 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -42,6 +42,7 @@ import { getPreferredSystemLanguages, getPrimaryPasteEnabled, getWindowFullscreenState, + getWindowShown, openExternal, readMedia, openSystemSettings, @@ -103,6 +104,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handleSync(getPreferredSystemLanguages); yield* ipc.handleSync(getPrimaryPasteEnabled); yield* ipc.handleSync(getWindowFullscreenState); + yield* ipc.handleSync(getWindowShown); yield* ipc.handleSync(getLocalEnvironmentBootstraps); yield* ipc.handleSync(getLocalEnvironmentEnabled); yield* ipc.handle(setLocalEnvironmentEnabled); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index b7bb5e579572f..ee81f753b97f7 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -20,6 +20,7 @@ export const TRACKPAD_SCROLL_BEGIN_CHANNEL = "desktop:trackpad-scroll-begin"; export const TRACKPAD_SCROLL_END_CHANNEL = "desktop:trackpad-scroll-end"; export const GET_WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:get-window-fullscreen-state"; export const WINDOW_FULLSCREEN_STATE_CHANNEL = "desktop:window-fullscreen-state"; +export const GET_WINDOW_SHOWN_CHANNEL = "desktop:get-window-shown"; export const SYNC_HTML_FULLSCREEN_CHANNEL = "desktop:sync-html-fullscreen"; export const DESKTOP_APP_ACTIVATION_READY_CHANNEL = "desktop:app-activation-ready"; export const DESKTOP_APP_ACTIVATION_COMPLETE_CHANNEL = "desktop:app-activation-complete"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index cc810fee072ad..59469d3769d18 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -84,9 +84,25 @@ const createTab = DesktopIpc.makeIpcMethod({ zoomFactor, colorScheme, serverTab, + background, }) { const manager = yield* PreviewManager.PreviewManager; - yield* manager.createTab(tabId, { zoomFactor, colorScheme, serverTab }); + if (background === undefined) { + yield* manager.createTab(tabId, { zoomFactor, colorScheme, serverTab }); + return; + } + const { scope, persistent, namespace } = resolvePartitionScope( + background.environmentId, + background.profileId, + ); + // Loading the session installs its UA rewrite and permission handlers. + yield* manager.getBrowserSession(scope, persistent, namespace); + const partition = yield* manager.getBrowserPartition(scope, persistent, namespace); + yield* manager.createTab( + tabId, + { zoomFactor, colorScheme, serverTab }, + { partition, url: background.url }, + ); }), }); diff --git a/apps/desktop/src/ipc/methods/window.ts b/apps/desktop/src/ipc/methods/window.ts index 3a525e8eac420..9756344a3a715 100644 --- a/apps/desktop/src/ipc/methods/window.ts +++ b/apps/desktop/src/ipc/methods/window.ts @@ -136,6 +136,16 @@ export const getWindowFullscreenState = DesktopIpc.makeSyncIpcMethod({ }), }); +export const getWindowShown = DesktopIpc.makeSyncIpcMethod({ + channel: IpcChannels.GET_WINDOW_SHOWN_CHANNEL, + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.window.getWindowShown")(function* () { + const electronWindow = yield* ElectronWindow.ElectronWindow; + const window = yield* electronWindow.currentMainOrFirst; + return Option.isSome(window) && window.value.isVisible() && !window.value.isMinimized(); + }), +}); + export const getLocalEnvironmentBootstraps = DesktopIpc.makeSyncIpcMethod({ channel: IpcChannels.GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL, result: Schema.Array(DesktopEnvironmentBootstrapSchema), diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 0dacf5ecaabf2..bb96f70fcc19a 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -306,6 +306,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { }, getWindowFullscreenState: () => ipcRenderer.sendSync(IpcChannels.GET_WINDOW_FULLSCREEN_STATE_CHANNEL) === true, + getWindowShown: () => ipcRenderer.sendSync(IpcChannels.GET_WINDOW_SHOWN_CHANNEL) !== false, getWindowZoomFactor: () => webFrame.getZoomFactor(), onWindowFullscreenStateChange: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, fullscreen: unknown) => { @@ -382,6 +383,7 @@ contextBridge.exposeInMainWorld("desktopBridge", { zoomFactor: defaults?.zoomFactor, serverTab: defaults?.serverTab, colorScheme: defaults?.colorScheme, + background: defaults?.background, }), closeTab: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_CLOSE_TAB_CHANNEL, { tabId }), registerWebview: (tabId, webContentsId) => diff --git a/apps/desktop/src/preview/BackgroundTabWindow.ts b/apps/desktop/src/preview/BackgroundTabWindow.ts new file mode 100644 index 0000000000000..f9ceb472caf97 --- /dev/null +++ b/apps/desktop/src/preview/BackgroundTabWindow.ts @@ -0,0 +1,42 @@ +import type { BrowserWindowConstructorOptions } from "electron"; + +// Kept free of runtime imports: scripts/verify-background-preview.mjs loads it +// straight into Electron to prove these options still paint while the app +// window is hidden. + +// The server sizes the page through emulation; this only seeds the window. +const BACKGROUND_TAB_WIDTH = 1280; +const BACKGROUND_TAB_HEIGHT = 800; + +/** + * An agent tab's never-shown window. A panel `` that attaches while + * the app window is hidden or minimized never paints, so its screenshots never + * return; a window of its own paints in every app window state. + */ +export function backgroundTabWindowOptions(input: { + readonly partition: string; + readonly preload: string; + readonly passkeyBridgeArgument: string | undefined; +}): BrowserWindowConstructorOptions { + return { + width: BACKGROUND_TAB_WIDTH, + height: BACKGROUND_TAB_HEIGHT, + show: false, + paintWhenInitiallyHidden: true, + skipTaskbar: true, + webPreferences: { + partition: input.partition, + preload: input.preload, + // Mirrors `will-attach-webview` for panel guests. + sandbox: true, + nodeIntegration: false, + nodeIntegrationInSubFrames: false, + contextIsolation: false, + disableHtmlFullscreenWindowResize: true, + backgroundThrottling: false, + ...(input.passkeyBridgeArgument === undefined + ? {} + : { additionalArguments: [input.passkeyBridgeArgument] }), + }, + }; +} diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 296be2fa6709f..d48a549770390 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -51,6 +51,7 @@ const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000$ interface AttachedTab { readonly key: DesktopBrowserTabKey; readonly debuggee: DesktopBrowserTabDebugger; + readonly background: boolean; relay: CdpRelayConnection | null; screenshotTurn: object | null; /** Each keeps the guest painting while whoever queued it still waits, and focused while `held`. */ @@ -97,7 +98,12 @@ export class DesktopBrowserHost extends Context.Service< { readonly events: Stream.Stream; readonly handleCommandLine: (line: string) => Effect.Effect; - readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void; + /** `background` tabs live in a never-shown window that no panel sizes. */ + readonly attach: ( + key: DesktopBrowserTabKey, + debuggee: DesktopBrowserTabDebugger, + background?: boolean, + ) => void; readonly detach: (key: DesktopBrowserTabKey) => void; readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; @@ -485,13 +491,18 @@ export const make = Effect.gen(function* () { if (announced) emit({ type: "detached", ...key }); }; - const attach = (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => { + const attach = ( + key: DesktopBrowserTabKey, + debuggee: DesktopBrowserTabDebugger, + background = false, + ) => { const id = keyOf(key); if (tabs.get(id)?.debuggee.webContents === debuggee.webContents) return; detach(key); const tab: AttachedTab = { key, debuggee, + background, relay: null, screenshotTurn: null, waitingScreenshots: [], @@ -529,7 +540,7 @@ export const make = Effect.gen(function* () { debuggee.debugger.on("message", tab.onMessage); debuggee.webContents.on("did-navigate", tab.onNewDocument); const announce = () => { - emit({ type: "attached", ...key }); + emit({ type: "attached", ...key, ...(background ? { background } : {}) }); const shown = shownTabs.get(id); if (shown !== undefined) emit({ type: "shown", ...key, shown }); }; @@ -631,7 +642,11 @@ export const make = Effect.gen(function* () { // Announced once its first page is up. if (tab.stopWaiting !== null) return Effect.void; const shown = shownTabs.get(keyOf(tab.key)); - return PubSub.publish(outbox, { type: "attached", ...tab.key }).pipe( + return PubSub.publish(outbox, { + type: "attached", + ...tab.key, + ...(tab.background ? { background: true } : {}), + }).pipe( Effect.andThen( shown !== undefined ? PubSub.publish(outbox, { type: "shown", ...tab.key, shown }) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 802cd315b8e60..b2439d92c2f6d 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -242,6 +242,7 @@ const managerLayer = (platform: NodeJS.Platform = "darwin") => }), ), Layer.provideMerge(DesktopBrowserHost.layer), + Layer.provideMerge(ElectronWindow.layer), Layer.provideMerge(layerBrowserSession), Layer.provideMerge( Layer.mock(PreviewPasskeys.PreviewPasskeys)({ @@ -622,6 +623,7 @@ describe("PreviewManager", () => { agentLedPage: () => false, placeDownload: () => true, }), + ElectronWindow.layer, Layer.mock(PreviewPasskeys.PreviewPasskeys)({ bridgeEnabled: false, installSessionHandlers: () => {}, @@ -719,6 +721,66 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "hosts an agent tab in a never-shown window until a panel webview replaces it", + () => + withManager((manager) => + Effect.gen(function* () { + const tab = makeTestPreviewWebContents( + async () => ({ + toJPEG: () => Buffer.from(""), + getSize: () => ({ width: 1, height: 1 }), + }), + 55, + null as unknown as TestHostWebContents, + ); + const loadURL = vi.fn(async () => undefined); + Object.assign(tab, { getType: () => "window", once: vi.fn(), loadURL, reload: vi.fn() }); + const window = { + webContents: tab, + once: vi.fn(), + isDestroyed: () => false, + destroy: vi.fn(), + }; + let options: Electron.BrowserWindowConstructorOptions | undefined; + browserWindowConstructor.mockImplementation(function ( + next: Electron.BrowserWindowConstructorOptions, + ) { + options = next; + return window; + }); + fromId.mockReturnValue(tab); + + const state = yield* manager.createTab( + "tab_agent", + { serverTab: { threadId: "thread", tabId: "agent" } }, + { partition: "persist:t3code-preview-test", url: "https://example.com/agent" }, + ); + + expect(options).toMatchObject({ + show: false, + paintWhenInitiallyHidden: true, + webPreferences: { + partition: "persist:t3code-preview-test", + backgroundThrottling: false, + }, + }); + expect(state.webContentsId).toBe(55); + expect(loadURL).toHaveBeenCalledWith("https://example.com/agent"); + fromId.mockReturnValue( + makeTestPreviewWebContents(async () => ({ + toJPEG: () => Buffer.from(""), + getSize: () => ({ width: 1, height: 1 }), + })), + ); + yield* manager.registerWebview("tab_agent", 42); + expect(window.destroy).toHaveBeenCalledOnce(); + yield* manager.closeTab("tab_agent"); + expect(window.destroy).toHaveBeenCalledOnce(); + }), + ), + ); + effectIt.effect.each([ ["darwin", true, false, true], ["darwin", false, true, false], diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index eba18ab631dd0..f7e78295b4ec6 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -69,8 +69,11 @@ import { PREVIEW_ANNOTATION_VOICE_EVENT_CHANNEL, PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, } from "../ipc/channels.ts"; +import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as DesktopBrowserHost from "./DesktopBrowserHost.ts"; import * as PreviewPasskeys from "./Passkeys.ts"; +import { PASSKEY_BRIDGE_ARGUMENT } from "./GuestProtocol.ts"; +import { backgroundTabWindowOptions } from "./BackgroundTabWindow.ts"; import * as BrowserSession from "./BrowserSession.ts"; import { ANNOTATION_CAPTURED_CHANNEL, @@ -123,6 +126,12 @@ export interface PreviewTabState { updatedAt: string; } +/** An agent tab's never-shown window: the main-process-derived partition it browses with. */ +export interface PreviewBackgroundTab { + readonly partition: string; + readonly url?: string | undefined; +} + const ZOOM_LEVELS: ReadonlyArray = [ 0.25, 0.33, 0.5, 0.67, 0.75, 0.8, 0.9, 1.0, 1.1, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 4.0, 5.0, ]; @@ -559,6 +568,7 @@ const isPreviewInputSignal = (value: unknown): value is PreviewInputSignal => { const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function* ( artifactDirectory: string, pictureInPicturePreloadPath: string, + guestPreloadPath: string, ) { const fileSystem = yield* FileSystem.FileSystem; const rendererHistory = yield* DesktopRendererHistory.DesktopRendererHistory; @@ -569,6 +579,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const browserHost = yield* DesktopBrowserHost.DesktopBrowserHost; const browserSession = yield* BrowserSession.BrowserSession; const passkeys = yield* PreviewPasskeys.PreviewPasskeys; + const electronWindow = yield* ElectronWindow.ElectronWindow; const context = yield* Effect.context(); const runFork = Effect.runForkWith(context); const runPromise = Effect.runPromiseWith(context); @@ -1707,6 +1718,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (currentMainWindow !== window) return; currentMainWindow = undefined; frameCaptureWindowOpen = false; + // Off macOS the app quits once its last window closes, which a + // background tab would otherwise hold open. + if (hostPlatform !== "darwin") { + for (const tabId of backgroundWindows.keys()) closeBackgroundWindow(tabId); + } mainWindowCleanupFiber = runFork( Effect.all([closeAllPictureInPicture(), stopAllRecordings()], { concurrency: "unbounded", @@ -1765,11 +1781,74 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return result.state; }); + /** Agent tabs' never-shown windows; see backgroundTabWindowOptions. */ + const backgroundWindows = new Map(); + + const openBackgroundWindow = Effect.fn("PreviewManager.openBackgroundWindow")(function* ( + tabId: string, + background: PreviewBackgroundTab, + ) { + // Auxiliary, so activating the app never picks it as the main window. + const window = yield* electronWindow + .create( + backgroundTabWindowOptions({ + partition: background.partition, + preload: guestPreloadPath, + passkeyBridgeArgument: passkeys.bridgeEnabled ? PASSKEY_BRIDGE_ARGUMENT : undefined, + }), + { auxiliary: true }, + ) + .pipe( + Effect.mapError( + (cause) => + new PreviewOperationError({ operation: "backgroundWindow.create", tabId, cause }), + ), + ); + backgroundWindows.set(tabId, window); + const wc = window.webContents; + window.once("closed", () => { + if (backgroundWindows.get(tabId) === window) backgroundWindows.delete(tabId); + }); + // A panel guest is remounted by the renderer after a crash; nothing else + // would bring this one back. + wc.on("render-process-gone", () => { + if (!wc.isDestroyed()) wc.reload(); + }); + yield* prepareWebview(wc); + yield* registerWebviewUnlocked(tabId, wc.id, tabLifecycleGenerations.get(tabId)); + // A navigation that arrived during setup is newer; registration already loads it. + const navStatus = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.navStatus; + if (background.url !== undefined && navStatus?.kind !== "Loading") { + runFork( + attemptPromise({ operation: "backgroundWindow.load", tabId, webContentsId: wc.id }, () => + wc.loadURL(background.url!), + ).pipe(Effect.ignore), + ); + } + }); + + const closeBackgroundWindow = (tabId: string) => { + const window = backgroundWindows.get(tabId); + backgroundWindows.delete(tabId); + if (window && !window.isDestroyed()) window.destroy(); + }; + const createTab = Effect.fn("PreviewManager.createTab")(function* ( tabId: string, defaults?: DesktopPreviewTabDefaults, + background?: PreviewBackgroundTab, ) { - return yield* withTabLifecycleLock(tabId, createTabUnlocked(tabId, defaults)); + return yield* withTabLifecycleLock( + tabId, + Effect.gen(function* () { + const state = yield* createTabUnlocked(tabId, defaults); + if (background === undefined || backgroundWindows.has(tabId)) return state; + yield* openBackgroundWindow(tabId, background).pipe( + Effect.onError(() => Effect.sync(() => closeBackgroundWindow(tabId))), + ); + return (yield* SynchronizedRef.get(tabsRef)).get(tabId) ?? state; + }), + ); }); const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { @@ -1809,6 +1888,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { concurrency: 2, discard: true }, ); } + closeBackgroundWindow(tabId); const updatedAt = yield* currentIso; const closed: PreviewTabState = { ...closedTab, @@ -1861,11 +1941,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } const wc = webContents.fromId(webContentsId); const mainWindow = yield* Ref.get(mainWindowRef); + const ownBackgroundWindow = backgroundWindows.get(tabId)?.webContents === wc; if ( !wc || wc.isDestroyed() || - wc.getType() !== "webview" || - (Option.isSome(mainWindow) && wc.hostWebContents !== mainWindow.value.webContents) + (!ownBackgroundWindow && + (wc.getType() !== "webview" || + (Option.isSome(mainWindow) && wc.hostWebContents !== mainWindow.value.webContents))) ) { return yield* new PreviewWebContentsNotFoundError({ tabId, webContentsId }); } @@ -2003,7 +2085,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const expectedGeneration = tabLifecycleGenerations.get(tabId); return yield* withTabLifecycleLock( tabId, - registerWebviewUnlocked(tabId, webContentsId, expectedGeneration), + registerWebviewUnlocked(tabId, webContentsId, expectedGeneration).pipe( + // A reloaded renderer may host a background tab in a panel webview instead. + Effect.tap(() => Effect.sync(() => closeBackgroundWindow(tabId))), + ), ); }); @@ -2528,11 +2613,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } if (afterAttach.serverTab) { - browserHost.attach(afterAttach.serverTab, { - webContents: wc, - debugger: control.debugger, - cookiesCleared: () => runPromise(browserSession.cookiesCleared(wc.session)), - }); + browserHost.attach( + afterAttach.serverTab, + { + webContents: wc, + debugger: control.debugger, + cookiesCleared: () => runPromise(browserSession.cookiesCleared(wc.session)), + }, + backgroundWindows.get(tabId)?.webContents === wc, + ); // CDP cannot recover a failure that happened before it attached. const { navStatus } = afterAttach; if (navStatus.kind === "LoadFailed") { @@ -3773,6 +3862,7 @@ export class PreviewManager extends Context.Service< readonly createTab: ( tabId: string, defaults?: DesktopPreviewTabDefaults, + background?: PreviewBackgroundTab, ) => Effect.Effect; readonly closeTab: (tabId: string) => Effect.Effect; readonly registerWebview: ( @@ -3931,6 +4021,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { const operations = yield* makeNativeOperations( environment.browserArtifactsDir, environment.path.join(environment.dirname, "preview-pip-preload.cjs"), + environment.path.join(environment.dirname, "preview-pick-preload.cjs"), ); return PreviewManager.of({ diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index 5d6b55fa882e8..400403d080cbf 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -45,6 +45,8 @@ export class DesktopBrowserChannel extends Context.Service< readonly detached: Stream.Stream; readonly attached: Stream.Stream; readonly isAttached: (key: DesktopTabKey) => Effect.Effect; + /** The tab lives in a never-shown desktop window, so no panel sizes it. */ + readonly isBackground: (key: DesktopTabKey) => boolean; /** Whether a desktop window draws the tab on screen, which no streamed viewer counts. */ readonly isShown: (key: DesktopTabKey) => boolean; /** A load failure the desktop saw, possibly before the server connected to the tab. */ @@ -64,7 +66,8 @@ const make = Effect.gen(function* () { const inputFd = config.desktopBrowserFd; const controlFd = config.desktopBrowserControlFd; const changes = yield* PubSub.unbounded<{ key: DesktopTabKey; attached: boolean }>(); - const attachedTabs = new Set(); + /** Attached tab ids, mapped to whether each is hosted in the background. */ + const attachedTabs = new Map(); // Missing reports protect desktop tabs until the renderer explicitly marks them hidden. const shownTabs = new Map(); const loadFailures = new Map(); @@ -91,6 +94,7 @@ const make = Effect.gen(function* () { pointer: Effect.fn("DesktopBrowserChannel.pointer")(function () { return Effect.void; }), + isBackground: () => false, isShown: () => false, loadFailure: () => undefined, }); @@ -129,7 +133,7 @@ const make = Effect.gen(function* () { return queue ? Queue.offer(queue, event.message).pipe(Effect.asVoid) : Effect.void; } case "attached": - attachedTabs.add(id); + attachedTabs.set(id, event.background === true); return PubSub.publish(changes, { key, attached: true }); case "loadFailed": loadFailures.set(id, { @@ -252,6 +256,7 @@ const make = Effect.gen(function* () { isAttached: Effect.fn("DesktopBrowserChannel.isAttached")(function (key) { return Effect.sync(() => attachedTabs.has(keyOf(key))); }), + isBackground: (key) => attachedTabs.get(keyOf(key)) === true, isShown: (key) => shownTabs.get(keyOf(key)) ?? true, loadFailure: (key) => loadFailures.get(keyOf(key)), endpoint, diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index d59738624f416..1510890b29820 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -395,6 +395,7 @@ let contextFailure: Error | null = null; let pageSetupFailure: Error | null = null; let initialNavigation: (() => Promise) | null = null; let desktopRendersNext = false; +let desktopBackground = false; let desktopPageUrl = "about:blank"; /** Replaces the answer to the desktop page's size reads. */ let desktopSizeAnswer: (() => Promise>) | null = null; @@ -499,6 +500,7 @@ const dependencies = Layer.mergeAll( ), ), isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), + isBackground: () => desktopBackground, isShown: (key) => shownDesktopTabs.has(key.tabId), loadFailure: (key) => desktopLoadFailures.get(key.tabId), endpoint: (key) => @@ -584,6 +586,7 @@ beforeEach(() => { shownDesktopTabs.clear(); desktopLoadFailures.clear(); desktopRendersNext = false; + desktopBackground = false; desktopAttachesLate = false; desktopAttachWaiting = Promise.withResolvers(); desktopPageUrl = "about:blank"; @@ -1513,6 +1516,28 @@ it.effect("enforces provider ownership and explicit targets when a session has m ).pipe(Effect.provide(layer)), ); +it.effect.each([ + { background: true, sized: true }, + { background: false, sized: false }, +])( + "a desktop tab is sized by the server only when no panel webview sizes it (background $background)", + ({ background, sized }) => + Effect.scoped( + Effect.gen(function* () { + desktopRendersNext = true; + desktopBackground = background; + const { browser, tabId } = yield* ready; + const page = desktopConnections[0]!.context.page; + page.setViewportSize.mockClear(); + const operator = yield* browser.attachViewer(viewerInput(tabId, true)); + yield* operator.input({ type: "takeControl" }); + yield* operator.input({ type: "resize", width: 390, height: 844 }); + if (sized) expect(page.setViewportSize).toHaveBeenCalledWith({ width: 390, height: 844 }); + else expect(page.setViewportSize).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + it.effect("a first still that never returns does not stall the tab's later captures", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index c87e829130e25..959ad02b11cda 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -349,7 +349,12 @@ interface ServerTab { readonly isolatedContext: boolean; - readonly desktop: { readonly close: () => Promise } | null; + /** + * A desktop-rendered page. `background` pages live in a never-shown window, + * so the server sizes them like headless pages; the others follow the + * panel's ``. + */ + readonly desktop: { readonly close: () => Promise; readonly background: boolean } | null; readonly profileId: string | undefined; readonly openerTabId: string | undefined; @@ -644,9 +649,11 @@ const make = Effect.gen(function* () { } : null; + const inPanel = (tab: ServerTab) => tab.desktop !== null && !tab.desktop.background; + const updateFileChooserInterception = (tab: ServerTab) => { // Only agents and streamed controllers can answer intercepted uploads; native desktop views use the OS picker. - const intercept = !tab.desktop || tab.agentOperating || tab.control.controller !== null; + const intercept = !inPanel(tab) || tab.agentOperating || tab.control.controller !== null; if (intercept === tab.fileChooserIntercepted) return; tab.fileChooserIntercepted = intercept; if (intercept) tab.page.on("filechooser", tab.onFileChooser); @@ -784,7 +791,7 @@ const make = Effect.gen(function* () { const layoutSize = async (tab: ServerTab) => { // A dialog blocks page script; its close measures again. - if (tab.desktop) { + if (inPanel(tab)) { return tab.dialog ? tab.desktopViewport : ((await measureDesktopViewport(tab)) ?? tab.desktopViewport); @@ -807,13 +814,10 @@ const make = Effect.gen(function* () { }; const applyZoom = async (tab: ServerTab) => { - if (tab.desktop) return; + if (inPanel(tab)) return; const size = tab.page.viewportSize(); if (!size) return; - if (tab.zoomFactor === 1) { - await tab.page.setViewportSize(size); - return; - } + // Playwright skips a size it already set, so 100% must replace the zoomed override itself. const layout = (await layoutSize(tab))!; await tab.cdp.send("Emulation.setDeviceMetricsOverride", { width: layout.width, @@ -832,7 +836,7 @@ const make = Effect.gen(function* () { const applySetting = async (tab: ServerTab, setting: PreviewViewportSetting) => { tab.setting = setting; - if (tab.desktop) return; + if (inPanel(tab)) return; const size = fixedViewportSize(setting) ?? [...tab.viewers] @@ -852,7 +856,7 @@ const make = Effect.gen(function* () { await tab.page.emulateMedia({ colorScheme: colorScheme === "system" ? null : colorScheme }); } const zoomFactor = snapshot.zoomFactor ?? 1; - if (zoomFactor !== tab.zoomFactor && !tab.desktop) { + if (zoomFactor !== tab.zoomFactor && !inPanel(tab)) { tab.zoomFactor = zoomFactor; await applyZoom(tab); await broadcastViewport(tab); @@ -949,6 +953,10 @@ const make = Effect.gen(function* () { const connected = await contexts.connectDesktopPage(endpoint); return { page: connected.page, + background: desktopChannel.isBackground({ + threadId: snapshot.threadId, + tabId: snapshot.tabId, + }), close: async () => { await connected.browser.close().catch(constVoid); await runPromise(Scope.close(scope, Exit.void)); @@ -1049,7 +1057,7 @@ const make = Effect.gen(function* () { .catch(constVoid); }, isolatedContext, - desktop: desktop === null ? null : { close: desktop.close }, + desktop: desktop === null ? null : { close: desktop.close, background: desktop.background }, profileId: snapshot.profileId, openerTabId: adopted?.openerTabId, downloads: [], @@ -1087,7 +1095,7 @@ const make = Effect.gen(function* () { captureLock: Promise.resolve(), capturing: 0, }; - if (!desktop) { + if (!inPanel(tab)) { await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); } await applyRendering(tab, snapshot); @@ -1656,7 +1664,7 @@ const make = Effect.gen(function* () { // The desktop draws its tabs in its own webview, so they never get a streamed viewer. const isVisible = (tab: ServerTab) => - tab.viewers.size > 0 || (tab.desktop !== null && desktopChannel.isShown(tab)); + tab.viewers.size > 0 || (inPanel(tab) && desktopChannel.isShown(tab)); const idleAgentTabs = () => { const cutoff = Date.now() - AGENT_TAB_IDLE_MS; @@ -1724,10 +1732,10 @@ const make = Effect.gen(function* () { ); }, Effect.uninterruptible); - // Desktop browsing bypasses streamed viewers and server input tracking, - // so these idle signals only apply to headless tabs. + // Panel browsing bypasses streamed viewers and server input tracking, + // so these idle signals only apply to tabs the panel does not render. const closeIdleAgentTabs = () => { - for (const tab of idleAgentTabs()) if (tab.desktop === null) dropTab(tab, true); + for (const tab of idleAgentTabs()) if (!inPanel(tab)) dropTab(tab, true); }; const latestThreadTab = (threadId: string, agentSessionId?: string) => { @@ -1953,7 +1961,7 @@ const make = Effect.gen(function* () { const statusUrl = loadError?.url ?? (url === "about:blank" ? null : url); const [viewport, title] = await Promise.all([ // A late measurement still lands in desktopViewport for the next status. - tab.desktop + inPanel(tab) ? withinStatusDeadline(desktopStatusViewport(tab), () => tab.desktopViewport) : tab.page.viewportSize(), statusUrl === null || tab.dialog @@ -2276,12 +2284,12 @@ const make = Effect.gen(function* () { async ({ x, y }, phase) => { const encoder = tab.recording?.encoder; - if (tab.viewers.size === 0 && !encoder && !tab.desktop) return; + if (tab.viewers.size === 0 && !encoder && !inPanel(tab)) return; const show = (next: "move" | "click") => { const sequence = ++pointerSequence; for (const viewer of tab.viewers) viewer.push({ _tag: "pointer", phase: next, x, y, sequence }); - if (tab.desktop) + if (inPanel(tab)) runFork( desktopChannel.pointer( { threadId: tab.threadId, tabId: tab.tabId }, @@ -2504,7 +2512,7 @@ const make = Effect.gen(function* () { }), ); // Only an explicit show is sure to reach a client; desktop tabs are visible without a viewer. - if (reveal === true && tab.viewers.size === 0 && tab.desktop === null) + if (reveal === true && tab.viewers.size === 0 && !inPanel(tab)) shown = awaitViewer( tab, Math.min(REVEAL_VIEWER_WAIT_MS, deadlineMs - performance.now()), @@ -2652,7 +2660,7 @@ const make = Effect.gen(function* () { manager.resize({ threadId: tab.threadId, tabId: tab.tabId, viewport: setting }), ); await applySetting(tab, setting); - if (!tab.desktop) { + if (!inPanel(tab)) { return { tabId: tab.tabId, setting, @@ -2804,7 +2812,7 @@ const make = Effect.gen(function* () { } if (event.type === "resized") { // Desktop tabs only record the setting, so status need not wait behind an agent action. - if (tab.desktop) tab.setting = event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT; + if (inPanel(tab)) tab.setting = event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT; await tab.control .system(async () => { if ( @@ -2943,10 +2951,11 @@ const make = Effect.gen(function* () { const height = Math.min(Math.round(num(message.height)), 2160); if (width < 100 || height < 100) return; viewer.requestedSize = { width, height, order: ++viewerResizeOrder }; - if (tab.setting._tag !== "fill" || tab.desktop) return; + if (tab.setting._tag !== "fill" || inPanel(tab)) return; const current = tab.page.viewportSize(); if (current?.width === width && current.height === height) return; await tab.page.setViewportSize({ width, height }); + await applyZoom(tab); await broadcastViewport(tab); return; } diff --git a/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx b/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx index 29c217e064adc..dac2848f4499e 100644 --- a/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.browser.test.tsx @@ -22,6 +22,15 @@ const mocks = vi.hoisted(() => ({ vi.mock(import("~/env"), async (original) => ({ ...(await original()), isElectron: true })); +// Server tabs render on the desktop only for its own environment. +vi.mock(import("~/state/primaryEnvironment"), async (original) => { + const { Atom } = await import("effect/reactivity"); + return { + ...(await original()), + primaryEnvironmentIdAtom: Atom.make(EnvironmentId.make("host-webview-order")), + }; +}); + vi.mock(import("~/state/session"), async (original) => ({ ...(await original()), useEnvironmentScope: () => true, @@ -58,6 +67,7 @@ import { import { reconcilePreviewServerSessions, resetPreviewStateForTests } from "~/previewStateStore"; import { AppAtomRegistryProvider } from "~/rpc/atomRegistry"; import { ElectronBrowserHost } from "./ElectronBrowserHost"; +import { rendersServerTabNatively } from "./previewRuntime"; const threadRef = scopeThreadRef( EnvironmentId.make("host-webview-order"), @@ -93,6 +103,15 @@ function setup(): Disposable { }; } +function stubDesktopWindowShown(shown: boolean): Disposable { + window.desktopBridge = { getWindowShown: () => shown, getWindowZoomFactor: () => 1 } as never; + return { + [Symbol.dispose]() { + delete window.desktopBridge; + }, + }; +} + async function renderHost() { await using stack = new AsyncDisposableStack(); const screen = stack.use( @@ -160,4 +179,55 @@ describe("ElectronBrowserHost", () => { expect(after).toHaveLength(2); expect(after.every((node, index) => node.isSameNode(before[index]!))).toBe(true); }); + + it.each([ + { shown: false, tabId: "tab_agent_hidden" }, + { shown: true, tabId: "tab_agent_shown" }, + ])( + "hosts an agent's tab in a background window only if the app window is hidden (shown $shown)", + async ({ shown, tabId }) => { + using _state = setup(); + using _bridge = stubDesktopWindowShown(shown); + const agentTab: PreviewSessionSnapshot = { + ...snapshot(tabId, "2026-01-01T00:00:01.000Z"), + runtime: "server", + automationOwner: "agent-session", + }; + reconcilePreviewServerSessions(threadRef, { + sessions: [agentTab], + serverEpoch: "server-a", + revision: 1, + }); + await using screen = await renderHost(); + + await vi.waitFor(() => + expect(mocks.createTab).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + serverTab: { threadId: threadRef.threadId, tabId }, + }), + ), + ); + const [, defaults] = mocks.createTab.mock.calls[0]!; + if (shown) { + expect(defaults?.background).toBeUndefined(); + expect(screen.container.querySelectorAll("webview")).toHaveLength(1); + } else { + expect(defaults?.background).toMatchObject({ + environmentId: threadRef.environmentId, + url: `https://example.com/${tabId}`, + }); + expect(screen.container.querySelectorAll("webview")).toHaveLength(0); + // Moving hosts would reload the agent's page, so it stays streamed once T3 is back. + window.desktopBridge = { + getWindowShown: () => true, + getWindowZoomFactor: () => 1, + } as never; + expect( + rendersServerTabNatively(threadRef.environmentId, threadRef.environmentId, agentTab), + ).toBe(false); + expect(screen.container.querySelectorAll("webview")).toHaveLength(0); + } + }, + ); }); diff --git a/apps/web/src/browser/ElectronBrowserHost.tsx b/apps/web/src/browser/ElectronBrowserHost.tsx index d71b7a2b6b2ba..c2cb6045102ec 100644 --- a/apps/web/src/browser/ElectronBrowserHost.tsx +++ b/apps/web/src/browser/ElectronBrowserHost.tsx @@ -5,10 +5,11 @@ import { type DesktopPreviewOpenLinkEvent, type EnvironmentId, FILL_PREVIEW_VIEWPORT, + type ScopedThreadRef, } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import { runAtomCommand } from "@t3tools/client-runtime/state/runtime"; -import { type ComponentProps, useState } from "react"; +import { type ComponentProps, useEffect, useState } from "react"; import { primaryEnvironmentIdAtom } from "~/state/primaryEnvironment"; @@ -29,7 +30,8 @@ import { useBrowserDefaults } from "./browserDefaults"; import { useBrowserPointerStore } from "./browserPointerStore"; import { HostedBrowserWebview } from "./HostedBrowserWebview"; import { openUrlInPreview } from "./openFileInPreview"; -import { rendersServerTabNatively } from "./previewRuntime"; +import { acquireDesktopTab } from "./desktopTabLifetime"; +import { hostsServerTabInBackground, rendersServerTabNatively } from "./previewRuntime"; import { previewRuntimeTabId } from "./previewRuntimeTabId"; /** Every session this window renders as a hosted webview, in a fixed (non-display) order. */ @@ -44,11 +46,17 @@ function hostedBrowserSessions( .filter( (snapshot) => snapshot.runtime !== "server" || - rendersServerTabNatively(threadRef.environmentId, primaryEnvironmentId, snapshot), + rendersServerTabNatively(threadRef.environmentId, primaryEnvironmentId, snapshot) || + hostsServerTabInBackground(threadRef.environmentId, primaryEnvironmentId, snapshot), ) .map((snapshot) => ({ threadRef, snapshot, + background: hostsServerTabInBackground( + threadRef.environmentId, + primaryEnvironmentId, + snapshot, + ), runtimeTabId: previewRuntimeTabId(threadRef, previewState.serverEpoch, snapshot.tabId), pictureInPicture: previewState.desktopByTabId[snapshot.tabId]?.pictureInPicture ?? false, @@ -134,35 +142,90 @@ export function ElectronBrowserHost() { if (!isElectron) return null; return (
- {sessions.map(({ threadRef, snapshot, runtimeTabId, pictureInPicture, zoomFactor }) => { - const url = snapshot.navStatus._tag === "Idle" ? null : snapshot.navStatus.url; - return ( - - ); - })} + {sessions.map( + ({ threadRef, snapshot, runtimeTabId, pictureInPicture, zoomFactor, background }) => { + const url = snapshot.navStatus._tag === "Idle" ? null : snapshot.navStatus.url; + if (background) { + return ( + + ); + } + return ( + + ); + }, + )}
); } +/** Holds the desktop tab open in its own window; the panel streams it from the server. */ +function BackgroundBrowserTab(props: { + readonly threadRef: ScopedThreadRef; + readonly tabId: string; + readonly runtimeTabId: string; + readonly initialUrl: string | null; + readonly profileId: string | undefined; +}) { + const { tabId, runtimeTabId } = props; + const { environmentId, threadId } = props.threadRef; + const canOperatePreview = useEnvironmentScope(environmentId, AuthPreviewOperateScope); + const profileId = useTabProfileId(props.profileId); + const hydrated = useClientSettingsHydrated(); + // Fixed at creation, like a webview's partition and first URL. + const [initialUrl] = useState(props.initialUrl); + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- leases the desktop's background window for this tab and releases it on cleanup + useEffect(() => { + if (!canOperatePreview || !hydrated) return; + const lease = acquireDesktopTab( + runtimeTabId, + { threadId, tabId }, + { + environmentId, + ...(profileId === undefined ? {} : { profileId }), + ...(initialUrl === null ? {} : { url: initialUrl }), + }, + ); + return lease.release; + }, [ + canOperatePreview, + hydrated, + environmentId, + threadId, + tabId, + runtimeTabId, + profileId, + initialUrl, + ]); + return null; +} + function AuthorizedBrowserWebview(props: ComponentProps) { const canOperatePreview = useEnvironmentScope( props.threadRef.environmentId, diff --git a/apps/web/src/browser/desktopTabLifetime.ts b/apps/web/src/browser/desktopTabLifetime.ts index 6c94951a0cf34..f0d7105a40333 100644 --- a/apps/web/src/browser/desktopTabLifetime.ts +++ b/apps/web/src/browser/desktopTabLifetime.ts @@ -1,3 +1,5 @@ +import type { DesktopPreviewTabDefaults } from "@t3tools/contracts"; + import { previewBridge } from "~/components/preview/previewBridge"; import { browserDefaultTabState, resolveBrowserDefaults } from "./browserDefaults"; @@ -44,6 +46,7 @@ export interface AcquiredDesktopTab { export function acquireDesktopTab( tabId: string, serverTab?: { readonly threadId: string; readonly tabId: string }, + background?: DesktopPreviewTabDefaults["background"], ): AcquiredDesktopTab { const current = leases.get(tabId) ?? @@ -55,6 +58,7 @@ export function acquireDesktopTab( previewBridge?.createTab(tabId, { ...browserDefaultTabState(await resolveBrowserDefaults()), ...(serverTab === undefined ? {} : { serverTab }), + ...(background === undefined ? {} : { background }), }), ), } satisfies DesktopTabLease); diff --git a/apps/web/src/browser/previewRuntime.ts b/apps/web/src/browser/previewRuntime.ts index 7ca4a8a8f6995..46401812a63df 100644 --- a/apps/web/src/browser/previewRuntime.ts +++ b/apps/web/src/browser/previewRuntime.ts @@ -41,10 +41,16 @@ export function usePreviewAvailable(environmentId: EnvironmentId | null): boolea return isPreviewSupportedInRuntime() || serverBrowser; } -export function rendersServerTabNatively( +type RuntimeSnapshot = Pick< + PreviewSessionSnapshot, + "threadId" | "tabId" | "runtime" | "headless" | "automationOwner" +>; + +/** The desktop app hosts this server tab, in a panel `` or a background window. */ +function desktopHostsServerTab( environmentId: EnvironmentId, primaryEnvironmentId: EnvironmentId | null, - snapshot: Pick | null | undefined, + snapshot: RuntimeSnapshot | null | undefined, ): boolean { return ( isElectron && @@ -55,9 +61,48 @@ export function rendersServerTabNatively( ); } +// Fixed when a tab first appears: Electron cannot move a page between a window and a +// ``, so changing hosts later would reload it and lose the agent's live page. +const backgroundHostedTabs = new Map(); + +/** + * A panel `` created while the app window is hidden or minimized never paints, so an + * agent tab that appears then runs in a never-shown desktop window and the panel streams it. + */ +export function hostsServerTabInBackground( + environmentId: EnvironmentId, + primaryEnvironmentId: EnvironmentId | null, + snapshot: RuntimeSnapshot | null | undefined, +): boolean { + if ( + !snapshot || + snapshot.automationOwner === undefined || + !desktopHostsServerTab(environmentId, primaryEnvironmentId, snapshot) + ) { + return false; + } + const key = `${environmentId}\u0000${snapshot.threadId}\u0000${snapshot.tabId}`; + const decided = backgroundHostedTabs.get(key); + if (decided !== undefined) return decided; + const background = window.desktopBridge?.getWindowShown?.() === false; + backgroundHostedTabs.set(key, background); + return background; +} + +export function rendersServerTabNatively( + environmentId: EnvironmentId, + primaryEnvironmentId: EnvironmentId | null, + snapshot: RuntimeSnapshot | null | undefined, +): boolean { + return ( + desktopHostsServerTab(environmentId, primaryEnvironmentId, snapshot) && + !hostsServerTabInBackground(environmentId, primaryEnvironmentId, snapshot) + ); +} + export function useRendersServerTabNatively( environmentId: EnvironmentId, - snapshot: Pick | null | undefined, + snapshot: RuntimeSnapshot | null | undefined, ): boolean { return rendersServerTabNatively(environmentId, useAtomValue(primaryEnvironmentIdAtom), snapshot); } diff --git a/knip.jsonc b/knip.jsonc index e3589f643c325..e9503f75051af 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -93,6 +93,8 @@ "src/preview-pip-preload.ts!", "src/mac-permission-preload.ts!", "src/preview/Annotation.css!", + // verify-background-preview.mjs starts Electron with this entry by path. + "scripts/verify-background-preview.electron.cjs", ], // The injected runtime resolves Playwright by string; release tooling runs // electron-builder from this workspace, outside its package scripts. diff --git a/packages/contracts/src/desktopBrowser.ts b/packages/contracts/src/desktopBrowser.ts index 1fa572ca662ae..b8e39d6f95108 100644 --- a/packages/contracts/src/desktopBrowser.ts +++ b/packages/contracts/src/desktopBrowser.ts @@ -8,7 +8,15 @@ const TabKey = { }; export const DesktopBrowserEvent = Schema.Union([ - Schema.Struct({ type: Schema.Literal("attached"), ...TabKey }), + Schema.Struct({ + type: Schema.Literal("attached"), + ...TabKey, + /** + * Hosted in a never-shown window rather than a panel ``, so no + * panel sizes it and the server must. + */ + background: Schema.optional(Schema.Boolean), + }), /** Its `` went away: closed, crashed, swapped, or devtools took the debugger. */ Schema.Struct({ type: Schema.Literal("detached"), ...TabKey }), Schema.Struct({ type: Schema.Literal("cdp"), ...TabKey, message: Schema.String }), diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index c5f9e46bb2594..afcb1b9384325 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -959,12 +959,30 @@ export const DesktopPreviewCreateTabInputSchema = Schema.Struct({ serverTab: Schema.optional( Schema.Struct({ threadId: TrimmedNonEmptyString, tabId: TrimmedNonEmptyString }), ), + /** + * Host the tab in a never-shown window instead of a panel ``. Main + * derives the partition from these, as it does for webview config. + */ + background: Schema.optional( + Schema.Struct({ + environmentId: EnvironmentId, + profileId: Schema.optional(BrowserProfileId), + url: Schema.optional(Schema.String), + }), + ), }); export interface DesktopPreviewTabDefaults { readonly zoomFactor?: number | undefined; readonly colorScheme?: DesktopPreviewColorScheme | undefined; readonly serverTab?: { readonly threadId: string; readonly tabId: string } | undefined; + readonly background?: + | { + readonly environmentId: EnvironmentId; + readonly profileId?: string | undefined; + readonly url?: string | undefined; + } + | undefined; } export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ @@ -1149,6 +1167,8 @@ export interface DesktopBridge { onSnapShotEvent: (listener: (event: DesktopSnapShotEvent) => void) => () => void; onQuitShortcut: (listener: (event: QuitShortcutHintEvent) => void) => () => void; getWindowFullscreenState: () => boolean; + /** Whether the app window is on screen: not hidden and not minimized. */ + getWindowShown: () => boolean; onWindowFullscreenStateChange: (listener: (fullscreen: boolean) => void) => () => void; getWindowZoomFactor: () => number; getUpdateState: () => Promise; diff --git a/scripts/check-changed.ts b/scripts/check-changed.ts index c556c9c91bb32..1e21f3a8ae3b1 100644 --- a/scripts/check-changed.ts +++ b/scripts/check-changed.ts @@ -563,6 +563,7 @@ const checkChanged = Effect.fn("checkChanged")(function* (input: { build: outputs.build, rust: outputs.rust, "mobile native lint": outputs.mobile_native, + "desktop background preview": outputs.desktop_preview, "release smoke": outputs.release_smoke, }).flatMap(([job, runs]) => (runs ? [job] : [])); if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`);