From faa878926d320b2254067bc330a942a242929d6f Mon Sep 17 00:00:00 2001 From: Nick Roberts <9039456+nickrroberts@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:07:35 -0600 Subject: [PATCH 1/5] fix(desktop): preview_wait_for waits for a selector that is not there yet Playwright's injected querySelector returns undefined when nothing matches, and the match check compared with null, so every selector wait succeeded immediately. Found live: a wait on a missing selector returned in 11ms. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/Manager.test.ts | 63 ++++++++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 3 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 1af273e8622ab..2584decec6b6b 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -4153,6 +4153,69 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "waitFor times out while the selector has no match and resolves once it does", + () => + withManager((manager) => + Effect.gen(function* () { + const present = new Set(); + // Mirrors Playwright's injected script: querySelector returns undefined on no match. + const pageContext = NodeVM.createContext({ + document: { body: { innerText: "" } }, + location: { href: "http://127.0.0.1/" }, + __t3PlaywrightInjected: { + parseSelector: (selector: string) => selector, + querySelector: (selector: string) => (present.has(selector) ? {} : undefined), + }, + }); + const sendCommand = vi.fn(async (method: string, params?: Record) => + method === "Runtime.evaluate" + ? { + result: { + value: NodeVM.runInContext(String(params?.["expression"]), pageContext), + }, + } + : undefined, + ); + const webContents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 52), + { + isDevToolsOpened: () => false, + debugger: { + isAttached: () => false, + attach: vi.fn(), + detach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + }, + ); + fromId.mockReturnValue(webContents); + yield* manager.createTab("tab_wait_for"); + yield* manager.registerWebview("tab_wait_for", 52); + + const missing = yield* manager + .automationWaitFor("tab_wait_for", { selector: "#never-appears", timeoutMs: 300 }) + .pipe(Effect.flip, Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(500); + expect(yield* Fiber.join(missing)).toMatchObject({ + _tag: "PreviewAutomationTimeoutError", + }); + + const appears = yield* manager + .automationWaitFor("tab_wait_for", { selector: "#later", timeoutMs: 5_000 }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* TestClock.adjust(200); + present.add("css=#later"); + yield* TestClock.adjust(200); + yield* Fiber.join(appears); + }), + ), + ); + effectIt.effect("types in background webviews and enables native key input", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 83f88e324dcfa..25737a89500d6 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -4500,7 +4500,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function send, `(() => { try { - const selectorMatched = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, false) !== null; })()` : "true"}; + // Playwright's querySelector returns undefined, not null, when nothing matches. + const selectorMatched = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, false) != null; })()` : "true"}; const textMatched = ${ textJson ? `(document.body?.innerText || "").includes(${textJson})` : "true" }; From 23db47c305b5b91ca3eaac96d19f78b02c7ed4d7 Mon Sep 17 00:00:00 2001 From: Nick Roberts <9039456+nickrroberts@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:07:37 -0600 Subject: [PATCH 2/5] fix(browser): say why an opened preview is not visible visible only reports whether the human can see the tab now, so an open=true preview in a thread the human is not viewing came back visible=false with no explanation. Status and open now include presentation: on-screen, shown-in-thread, or background. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- .../components/preview/PreviewAutomationHosts.tsx | 13 ++++++++++++- .../components/preview/previewPresentation.test.ts | 13 +++++++++++++ .../src/components/preview/previewPresentation.ts | 13 +++++++++++++ packages/contracts/src/previewAutomation.ts | 11 +++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/components/preview/previewPresentation.test.ts create mode 100644 apps/web/src/components/preview/previewPresentation.ts diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 1c790e8b643ac..7e778c01ecba1 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -66,7 +66,7 @@ const PreviewStatusTool = Tool.make("preview_status", { const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", + "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. visible is true only while the human is looking at the tab; presentation says why it is not (shown-in-thread means the preview is open in this thread but the human is viewing something else). Automation works either way. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 173a027e9df26..f9a3d3e2ad6c8 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -64,6 +64,7 @@ import { useAtomQueryRunner } from "~/state/use-atom-query-runner"; import { useAtomCommand } from "~/state/use-atom-command"; import { previewBridge } from "./previewBridge"; +import { resolvePreviewPresentation } from "./previewPresentation"; import { PreviewAutomationOperationError, PreviewAutomationOverlayTimeoutError, @@ -237,6 +238,15 @@ const currentStatus = async ( const visible = runtimeTabId ? (useBrowserSurfaceStore.getState().byTabId[runtimeTabId]?.visible ?? false) : false; + const presentation = resolvePreviewPresentation({ + visible, + shownInThread: + tabId !== null && + selectThreadPreviewMiniPlayerTabId( + usePreviewMiniPlayerStore.getState().byThreadKey, + threadRef, + ) === tabId, + }); const renderingActive = runtimeTabId ? isPreviewWebviewRendering(runtimeTabId) : false; const viewportSetting = snapshot ? (snapshot.viewport ?? FILL_PREVIEW_VIEWPORT) : undefined; const viewport = @@ -249,12 +259,13 @@ const currentStatus = async ( }; if (runtimeTabId && tabId && previewBridge && state.desktopByTabId[tabId]) { const status = await previewBridge.automation.status(runtimeTabId); - return { ...status, tabId, visible, ...viewportStatus }; + return { ...status, tabId, visible, presentation, ...viewportStatus }; } const navStatus = snapshot?.navStatus; return { available: false, visible, + presentation, tabId, url: navStatus && navStatus._tag !== "Idle" ? navStatus.url : null, title: navStatus && navStatus._tag !== "Idle" ? navStatus.title : null, diff --git a/apps/web/src/components/preview/previewPresentation.test.ts b/apps/web/src/components/preview/previewPresentation.test.ts new file mode 100644 index 0000000000000..3f45c025566a1 --- /dev/null +++ b/apps/web/src/components/preview/previewPresentation.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { resolvePreviewPresentation } from "./previewPresentation"; + +describe("resolvePreviewPresentation", () => { + it("separates an off-screen thread preview from a background tab", () => { + expect(resolvePreviewPresentation({ visible: true, shownInThread: true })).toBe("on-screen"); + expect(resolvePreviewPresentation({ visible: false, shownInThread: true })).toBe( + "shown-in-thread", + ); + expect(resolvePreviewPresentation({ visible: false, shownInThread: false })).toBe("background"); + }); +}); diff --git a/apps/web/src/components/preview/previewPresentation.ts b/apps/web/src/components/preview/previewPresentation.ts new file mode 100644 index 0000000000000..3832e7d1f04b2 --- /dev/null +++ b/apps/web/src/components/preview/previewPresentation.ts @@ -0,0 +1,13 @@ +import type { PreviewAutomationStatus } from "@t3tools/contracts"; + +/** + * Explains a tab's `visible` flag to the agent. An `open: true` preview lives in + * its thread, so it is only on screen while the human is viewing that thread. + */ +export function resolvePreviewPresentation(input: { + readonly visible: boolean; + readonly shownInThread: boolean; +}): NonNullable { + if (input.visible) return "on-screen"; + return input.shownInThread ? "shown-in-thread" : "background"; +} diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index 551166784fced..e6bedf06d6cfc 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -74,6 +74,17 @@ export const PreviewAutomationStatus = Schema.Struct({ viewportSetting: Schema.optional(PreviewViewportSetting), /** Measured guest-page viewport in CSS pixels when a webview is ready. */ viewport: Schema.optional(PreviewRenderedViewportSize), + /** + * Why `visible` is what it is. `visible` only reports whether the human can see + * the tab right now, so an `open: true` tab in a thread the human is not looking + * at is "shown-in-thread" and not visible. Automation works in every state. + */ + presentation: Schema.optional( + Schema.Literals(["on-screen", "shown-in-thread", "background"]).annotate({ + description: + "on-screen: the human can see the tab now. shown-in-thread: it is the thread's open preview, but the human is viewing something else. background: not presented (open=false or closed by the human). Automation works in every state.", + }), + ), }); export type PreviewAutomationStatus = typeof PreviewAutomationStatus.Type; From 99448497f60e19a9a4baac3b2299a774c54d5d03 Mon Sep 17 00:00:00 2001 From: Nick Roberts <9039456+nickrroberts@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:07:39 -0600 Subject: [PATCH 3/5] fix(release): refresh DMG digests in latest-mac.yml after notarization electron-builder hashes the DMG before codesign and stapling rewrite it, so 0.0.66 shipped a manifest whose DMG sha512 and size did not match the published file. Recompute them after stapling. Co-Authored-By: Claude Opus 5.5 --- scripts/build-desktop-artifact.test.ts | 56 ++++++++++++++++++++++++-- scripts/build-desktop-artifact.ts | 47 +++++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/scripts/build-desktop-artifact.test.ts b/scripts/build-desktop-artifact.test.ts index ec057f4bbb042..61a2f282f50f2 100644 --- a/scripts/build-desktop-artifact.test.ts +++ b/scripts/build-desktop-artifact.test.ts @@ -3,7 +3,7 @@ import * as NodeCrypto from "node:crypto"; import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; +import { assert, describe, it } from "@effect/vitest"; import * as ConfigProvider from "effect/ConfigProvider"; import * as FileSystem from "effect/FileSystem"; import * as Effect from "effect/Effect"; @@ -24,6 +24,7 @@ import { LUCIVE_MAC_SIGN_IDENTITY, isMacDiskImageArtifact, resolveMacDmgCodesignIdentity, + refreshUpdateManifestFileEntry, signAndNotarizeMacDmgs, createStageWorkspaceConfig, createStagePatchedDependencies, @@ -2005,9 +2006,24 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { const dmg = path.join(outputDir, "T3-Code-0.0.42-arm64.dmg"); const blockmap = `${dmg}.blockmap`; const zip = path.join(outputDir, "T3-Code-0.0.42-arm64.zip"); + const manifest = path.join(outputDir, "latest-mac.yml"); yield* fs.writeFileString(dmg, "dmg"); yield* fs.writeFileString(blockmap, "blockmap"); yield* fs.writeFileString(zip, "zip"); + const manifestBefore = [ + "version: 0.0.42", + "files:", + " - url: T3-Code-0.0.42-arm64.zip", + " sha512: zip-hash", + " size: 3", + " - url: T3-Code-0.0.42-arm64.dmg", + " sha512: hash-before-signing", + " size: 999", + "path: T3-Code-0.0.42-arm64.zip", + "sha512: zip-hash", + "", + ].join("\n"); + yield* fs.writeFileString(manifest, manifestBefore); const commands: Array<{ readonly command: string; @@ -2028,7 +2044,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { const remaining = yield* signAndNotarizeMacDmgs({ signed: true, platform: "mac", - artifacts: [dmg, blockmap, zip], + artifacts: [dmg, blockmap, zip, manifest], verbose: false, }).pipe( Effect.provide(spawner), @@ -2045,8 +2061,18 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { ), ); - assert.deepStrictEqual(remaining, [dmg, zip]); + assert.deepStrictEqual(remaining, [dmg, zip, manifest]); assert.isFalse(yield* fs.exists(blockmap)); + // The manifest describes the DMG as shipped, after signing and stapling. + assert.strictEqual( + yield* fs.readFileString(manifest), + manifestBefore + .replace( + "hash-before-signing", + NodeCrypto.createHash("sha512").update("dmg").digest("base64"), + ) + .replace("size: 999", "size: 3"), + ); assert.deepStrictEqual( commands.map((command) => [command.command, command.args[0]]), [ @@ -2472,3 +2498,27 @@ it("ignores trailing separators", () => { ancestorNodeModulesPaths("C:\\tmp\\probe\\app", "\\"), ); }); + +describe("refreshUpdateManifestFileEntry", () => { + it("updates the top-level digest when the refreshed file is the primary path", () => { + const manifest = [ + "files:", + " - url: App.dmg", + " sha512: old", + " size: 1", + "path: App.dmg", + "sha512: old", + ].join("\n"); + assert.strictEqual( + refreshUpdateManifestFileEntry(manifest, "App.dmg", { sha512: "new", size: 2 }), + [ + "files:", + " - url: App.dmg", + " sha512: new", + " size: 2", + "path: App.dmg", + "sha512: new", + ].join("\n"), + ); + }); +}); diff --git a/scripts/build-desktop-artifact.ts b/scripts/build-desktop-artifact.ts index c60369303b0b7..2bdd2cc5b432a 100644 --- a/scripts/build-desktop-artifact.ts +++ b/scripts/build-desktop-artifact.ts @@ -1728,6 +1728,40 @@ export function isMacDiskImageArtifact(filePath: string): boolean { return filePath.endsWith(".dmg"); } +const isMacUpdateManifestArtifact = (filePath: string): boolean => + /(^|[\\/])[^\\/]*-mac\.yml$/u.test(filePath); + +/** + * Rewrites the sha512 and size recorded for `fileName` in an electron-updater + * manifest: its `files` entry and, when it is the manifest's primary `path`, the + * top-level sha512. Other entries and lines are left byte-for-byte. + */ +export function refreshUpdateManifestFileEntry( + manifest: string, + fileName: string, + digest: { readonly sha512: string; readonly size: number }, +): string { + const lines = manifest.split("\n"); + for (let index = 0; index < lines.length; index += 1) { + const entry = /^(\s*)- url: (.+)$/u.exec(lines[index] ?? ""); + if (!entry || entry[2]?.trim() !== fileName) continue; + const indent = `${entry[1] ?? ""} `; + for (let field = index + 1; field < lines.length; field += 1) { + const line = lines[field] ?? ""; + if (!line.startsWith(indent) || line.trimStart().startsWith("- ")) break; + if (line.trimStart().startsWith("sha512:")) + lines[field] = `${indent}sha512: ${digest.sha512}`; + if (line.trimStart().startsWith("size:")) lines[field] = `${indent}size: ${digest.size}`; + } + } + const primary = lines.findIndex((line) => line.trim() === `path: ${fileName}`); + if (primary !== -1) { + const topSha = lines.findIndex((line) => line.startsWith("sha512:")); + if (topSha !== -1) lines[topSha] = `sha512: ${digest.sha512}`; + } + return lines.join("\n"); +} + const MacDmgNotarizationConfig = Config.all({ appleId: Config.String("APPLE_ID").pipe(Config.option), applePassword: Config.String("APPLE_PASSWORD").pipe(Config.option), @@ -1805,6 +1839,19 @@ export const signAndNotarizeMacDmgs = Effect.fn("signAndNotarizeMacDmgs")(functi } const fs = yield* FileSystem.FileSystem; + // electron-builder hashed the DMGs before codesign and stapling rewrote them, so + // refresh their entries in any update manifest that lists them. + for (const manifest of input.artifacts.filter(isMacUpdateManifestArtifact)) { + let text = yield* fs.readFileString(manifest); + for (const dmg of dmgs) { + const bytes = yield* fs.readFile(dmg); + text = refreshUpdateManifestFileEntry(text, path.basename(dmg), { + sha512: NodeCrypto.createHash("sha512").update(bytes).digest("base64"), + size: bytes.byteLength, + }); + } + yield* fs.writeFileString(manifest, text); + } const remaining: string[] = []; for (const artifact of input.artifacts) { const staleBlockmap = dmgs.some((dmg) => artifact === `${dmg}.blockmap`); From d2ebc8d8246f5ebe6233bc340f2b2b3af0c9e86d Mon Sep 17 00:00:00 2001 From: Nick Roberts <9039456+nickrroberts@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:43:02 -0600 Subject: [PATCH 4/5] fix(desktop): background browser screenshots work while the app window is hidden Packaged builds hand the main window back to background throttling after first paint. When the window is minimized or occluded, an agent's offscreen browser tab then paints about once a second, so every capturePage attempt missed its 1s deadline: snapshots and screenshots failed while evaluate and click kept working. Still captures now lift main-window and guest throttling for their duration. The lease is counted together with recording and picture-in- picture sessions, so neither re-throttles under the other. Reproduced in an isolated dev build with packaged throttling and a minimized window: 0.0.65 failed every snapshot with 3x1s capturePage timeouts, this change captures in 7-40ms and restores throttling after. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/Manager.test.ts | 57 +++++++++++++++++- apps/desktop/src/preview/Manager.ts | 74 +++++++++++++++++++++++- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 2584decec6b6b..bb059dd7d46c5 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -3101,6 +3101,58 @@ describe("PreviewManager", () => { ), ); + effectIt.effect( + "lifts window and guest throttling while a still capture runs, without undoing a recording's", + () => + withManager((manager) => + Effect.gen(function* () { + // Packaged builds throttle the unfocused main window, and a throttled + // offscreen guest paints too slowly for capturePage's deadline. + let windowThrottled = true; + const windowThrottling = vi.fn((enabled: boolean) => { + windowThrottled = enabled; + }); + const throttlingAtCapture: Array<{ window: boolean; guest: boolean }> = []; + const capturePage = vi.fn(async () => { + throttlingAtCapture.push({ + window: windowThrottled, + guest: webContents.setBackgroundThrottling.mock.calls.at(-1)?.[0] ?? true, + }); + return { + toJPEG: () => Buffer.from("frame"), + toPNG: () => Buffer.from("png"), + getSize: () => ({ width: 1280, height: 720 }), + isEmpty: () => false, + }; + }); + const host = makeTestHostWebContents(); + const webContents = makeTestPreviewWebContents(capturePage, 42, host); + fromId.mockReturnValue(webContents); + yield* manager.createTab("tab_still_capture"); + yield* manager.registerWebview("tab_still_capture", 42); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { setBackgroundThrottling: windowThrottling }, + } as never); + + yield* manager.captureScreenshot("tab_still_capture"); + expect(throttlingAtCapture.at(-1)).toEqual({ window: false, guest: false }); + expect(windowThrottled).toBe(true); + expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([true]); + + // A recording keeps both unthrottled; a screenshot during it must not re-throttle. + yield* manager.startRecording("tab_still_capture"); + yield* manager.captureScreenshot("tab_still_capture"); + expect(windowThrottled).toBe(false); + expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([false]); + yield* manager.stopRecording("tab_still_capture"); + expect(windowThrottled).toBe(true); + expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([true]); + }), + ), + ); + effectIt.effect("continues native recording when the source warmup fails", () => withManager((manager) => Effect.gen(function* () { @@ -4199,7 +4251,10 @@ describe("PreviewManager", () => { const missing = yield* manager .automationWaitFor("tab_wait_for", { selector: "#never-appears", timeoutMs: 300 }) - .pipe(Effect.flip, Effect.forkChild({ startImmediately: true })); + .pipe( + Effect.match({ onFailure: (error) => error, onSuccess: () => "matched" as const }), + Effect.forkChild({ startImmediately: true }), + ); yield* TestClock.adjust(500); expect(yield* Fiber.join(missing)).toMatchObject({ _tag: "PreviewAutomationTimeoutError", diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 25737a89500d6..ea59b0a5118b6 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -669,6 +669,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const frameCaptureSessionsRef = yield* SynchronizedRef.make< ReadonlyMap >(new Map()); + // Still captures (snapshots, screenshots) in flight, by guest WebContents id. + // Packaged builds throttle the main window when it is not focused, which slows + // an offscreen guest to about one frame a second, so capturePage misses its + // deadline. Captures lift throttling while they run. Counting them alongside + // frame-capture sessions keeps either one from re-throttling under the other. + const stillCaptureLeasesRef = yield* Ref.make>(new Map()); + const stillCaptureLeaseTotal = Ref.get(stillCaptureLeasesRef).pipe( + Effect.map((leases) => Array.from(leases.values()).reduce((sum, count) => sum + count, 0)), + ); const pictureInPictureSessionsRef = yield* SynchronizedRef.make< ReadonlyMap >(new Map()); @@ -736,6 +745,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function schedule: Schedule.spaced(CAPTURE_PAGE_RETRY_DELAY_MS), while: isPreviewOperationError, }), + withStillCaptureUnthrottled(wc), ); }); const currentIso = DateTime.now.pipe(Effect.map(DateTime.formatIso)); @@ -849,6 +859,61 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); }); + const isHeldByFrameCapture = ( + sessions: ReadonlyMap, + webContentsId: number, + ) => + Array.from(sessions.values()).some((session) => + session.unthrottledWebContentsIds.has(webContentsId), + ); + const acquireStillCaptureUnthrottled = (wc: Electron.WebContents) => + SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => + Effect.gen(function* () { + const leases = yield* Ref.get(stillCaptureLeasesRef); + if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) { + yield* setFrameCaptureBackgroundThrottling(false).pipe(Effect.ignore); + } + if ((leases.get(wc.id) ?? 0) === 0 && !isHeldByFrameCapture(sessions, wc.id)) { + yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(Effect.ignore); + } + yield* Ref.set( + stillCaptureLeasesRef, + replaceMap(leases, (copy) => { + copy.set(wc.id, (copy.get(wc.id) ?? 0) + 1); + }), + ); + return [undefined, sessions] as const; + }), + ); + const releaseStillCaptureUnthrottled = (wc: Electron.WebContents) => + SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => + Effect.gen(function* () { + const leases = yield* Ref.get(stillCaptureLeasesRef); + const remaining = (leases.get(wc.id) ?? 1) - 1; + yield* Ref.set( + stillCaptureLeasesRef, + replaceMap(leases, (copy) => { + if (remaining > 0) copy.set(wc.id, remaining); + else copy.delete(wc.id); + }), + ); + if (remaining === 0 && !isHeldByFrameCapture(sessions, wc.id)) { + yield* setFrameCaptureWebContentsBackgroundThrottling(wc, true).pipe(Effect.ignore); + } + if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) { + yield* setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore); + } + return [undefined, sessions] as const; + }), + ).pipe(Effect.uninterruptible); + const withStillCaptureUnthrottled = + (wc: Electron.WebContents) => + (capture: Effect.Effect): Effect.Effect => + Effect.acquireUseRelease( + acquireStillCaptureUnthrottled(wc), + () => capture, + () => releaseStillCaptureUnthrottled(wc), + ); const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* ( tabId: string, consumer: FrameCaptureConsumer, @@ -878,10 +943,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const remainingSessions = replaceMap(sessions, (copy) => { copy.delete(tabId); }); + const stillCaptureLeases = yield* Ref.get(stillCaptureLeasesRef); yield* restoreFrameCaptureWebContentsBackgroundThrottling( - current.unthrottledWebContentsIds, + new Set( + Array.from(current.unthrottledWebContentsIds).filter( + (webContentsId) => (stillCaptureLeases.get(webContentsId) ?? 0) === 0, + ), + ), ); - if (remainingSessions.size === 0) { + if (remainingSessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) { yield* setFrameCaptureBackgroundThrottling(true).pipe( Effect.retry({ times: 2 }), Effect.catch((error) => From 63cb90023d579a35ff01509a0b4db281a8ddc3bf Mon Sep 17 00:00:00 2001 From: Nick Roberts <9039456+nickrroberts@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:59:53 -0600 Subject: [PATCH 5/5] fix(browser): address review of the background capture batch - Restore the main window's previous throttling after a still capture instead of forcing it on, so dev builds stay unthrottled. - Replacing the main window or failing a frame-capture start no longer re-throttles while a still capture holds its lease. - presentation counts a right-panel browser tab as shown in its thread, and only reports on-screen while the app window is visible. - preview_status documents presentation. Verified live: after a snapshot from a minimized window the guest goes back to zero frames and the window is throttled again. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/src/preview/Manager.test.ts | 33 +++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 19 +++++++++-- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- .../preview/PreviewAutomationHosts.tsx | 11 +++++-- .../preview/previewPresentation.test.ts | 20 ++++++++--- .../components/preview/previewPresentation.ts | 8 +++-- packages/contracts/src/previewAutomation.ts | 6 ++-- 7 files changed, 82 insertions(+), 17 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index bb059dd7d46c5..108b0281772ff 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -3153,6 +3153,39 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("leaves an unthrottled main window unthrottled after a still capture", () => + withManager((manager) => + Effect.gen(function* () { + // Dev builds keep the main window unthrottled so input stays responsive. + let windowThrottled = false; + const windowThrottling = vi.fn((enabled: boolean) => { + windowThrottled = enabled; + }); + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("frame"), + toPNG: () => Buffer.from("png"), + getSize: () => ({ width: 1280, height: 720 }), + isEmpty: () => false, + })); + const webContents = makeTestPreviewWebContents(capturePage, 42); + fromId.mockReturnValue(webContents); + yield* manager.createTab("tab_dev_capture"); + yield* manager.registerWebview("tab_dev_capture", 42); + yield* manager.setMainWindow({ + isDestroyed: () => false, + once: vi.fn(), + webContents: { + setBackgroundThrottling: windowThrottling, + getBackgroundThrottling: () => windowThrottled, + }, + } as never); + + yield* manager.captureScreenshot("tab_dev_capture"); + expect(windowThrottled).toBe(false); + }), + ), + ); + effectIt.effect("continues native recording when the source warmup fails", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index ea59b0a5118b6..71e8cc04089f8 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -675,6 +675,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // deadline. Captures lift throttling while they run. Counting them alongside // frame-capture sessions keeps either one from re-throttling under the other. const stillCaptureLeasesRef = yield* Ref.make>(new Map()); + // The main window's throttling before the first lease, restored after the last. + // Dev builds keep the window unthrottled on purpose; packaged builds throttle it. + const stillCaptureWindowThrottlingRef = yield* Ref.make(true); const stillCaptureLeaseTotal = Ref.get(stillCaptureLeasesRef).pipe( Effect.map((leases) => Array.from(leases.values()).reduce((sum, count) => sum + count, 0)), ); @@ -871,6 +874,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function Effect.gen(function* () { const leases = yield* Ref.get(stillCaptureLeasesRef); if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) { + const mainWindow = yield* Ref.get(mainWindowRef); + yield* Ref.set( + stillCaptureWindowThrottlingRef, + Option.isSome(mainWindow) && !mainWindow.value.isDestroyed() + ? mainWindow.value.webContents.getBackgroundThrottling?.() !== false + : true, + ); yield* setFrameCaptureBackgroundThrottling(false).pipe(Effect.ignore); } if ((leases.get(wc.id) ?? 0) === 0 && !isHeldByFrameCapture(sessions, wc.id)) { @@ -901,7 +911,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* setFrameCaptureWebContentsBackgroundThrottling(wc, true).pipe(Effect.ignore); } if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) { - yield* setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore); + yield* setFrameCaptureBackgroundThrottling( + yield* Ref.get(stillCaptureWindowThrottlingRef), + ).pipe(Effect.ignore); } return [undefined, sessions] as const; }), @@ -2118,7 +2130,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => Effect.gen(function* () { - if (sessions.size > 0) { + if (sessions.size > 0 || (yield* stillCaptureLeaseTotal) > 0) { yield* setWindowBackgroundThrottling(window, false); } yield* Ref.set(mainWindowRef, Option.some(window)); @@ -3184,9 +3196,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (sessions.size === 0) { yield* setFrameCaptureBackgroundThrottling(false); } + const stillCapturesActive = (yield* stillCaptureLeaseTotal) > 0; yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe( Effect.onError(() => - sessions.size === 0 + sessions.size === 0 && !stillCapturesActive ? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore) : Effect.void, ), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 7e778c01ecba1..282cadd0eb78f 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -52,7 +52,7 @@ const readonlyBrowserTool = (tool: T): T => const PreviewStatusTool = Tool.make("preview_status", { description: - "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", + "Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility (with presentation explaining why it is or is not on screen), loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index f9a3d3e2ad6c8..a5a181b044ad6 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -65,6 +65,7 @@ import { useAtomCommand } from "~/state/use-atom-command"; import { previewBridge } from "./previewBridge"; import { resolvePreviewPresentation } from "./previewPresentation"; +import { selectActiveRightPanelSurface, useRightPanelStore } from "~/rightPanelStore"; import { PreviewAutomationOperationError, PreviewAutomationOverlayTimeoutError, @@ -238,14 +239,20 @@ const currentStatus = async ( const visible = runtimeTabId ? (useBrowserSurfaceStore.getState().byTabId[runtimeTabId]?.visible ?? false) : false; + const rightPanelSurface = selectActiveRightPanelSurface( + useRightPanelStore.getState().byThreadKey, + threadRef, + ); const presentation = resolvePreviewPresentation({ visible, + windowVisible: document.visibilityState === "visible", shownInThread: tabId !== null && - selectThreadPreviewMiniPlayerTabId( + (selectThreadPreviewMiniPlayerTabId( usePreviewMiniPlayerStore.getState().byThreadKey, threadRef, - ) === tabId, + ) === tabId || + (rightPanelSurface?.kind === "preview" && rightPanelSurface.resourceId === tabId)), }); const renderingActive = runtimeTabId ? isPreviewWebviewRendering(runtimeTabId) : false; const viewportSetting = snapshot ? (snapshot.viewport ?? FILL_PREVIEW_VIEWPORT) : undefined; diff --git a/apps/web/src/components/preview/previewPresentation.test.ts b/apps/web/src/components/preview/previewPresentation.test.ts index 3f45c025566a1..cc5fdc287334d 100644 --- a/apps/web/src/components/preview/previewPresentation.test.ts +++ b/apps/web/src/components/preview/previewPresentation.test.ts @@ -4,10 +4,20 @@ import { resolvePreviewPresentation } from "./previewPresentation"; describe("resolvePreviewPresentation", () => { it("separates an off-screen thread preview from a background tab", () => { - expect(resolvePreviewPresentation({ visible: true, shownInThread: true })).toBe("on-screen"); - expect(resolvePreviewPresentation({ visible: false, shownInThread: true })).toBe( - "shown-in-thread", - ); - expect(resolvePreviewPresentation({ visible: false, shownInThread: false })).toBe("background"); + expect( + resolvePreviewPresentation({ visible: true, windowVisible: true, shownInThread: true }), + ).toBe("on-screen"); + expect( + resolvePreviewPresentation({ visible: false, windowVisible: true, shownInThread: true }), + ).toBe("shown-in-thread"); + expect( + resolvePreviewPresentation({ visible: false, windowVisible: true, shownInThread: false }), + ).toBe("background"); + }); + + it("does not claim on-screen while the app window is minimized or hidden", () => { + expect( + resolvePreviewPresentation({ visible: true, windowVisible: false, shownInThread: true }), + ).toBe("shown-in-thread"); }); }); diff --git a/apps/web/src/components/preview/previewPresentation.ts b/apps/web/src/components/preview/previewPresentation.ts index 3832e7d1f04b2..215f0ca8c0f02 100644 --- a/apps/web/src/components/preview/previewPresentation.ts +++ b/apps/web/src/components/preview/previewPresentation.ts @@ -2,12 +2,14 @@ import type { PreviewAutomationStatus } from "@t3tools/contracts"; /** * Explains a tab's `visible` flag to the agent. An `open: true` preview lives in - * its thread, so it is only on screen while the human is viewing that thread. + * its thread (floating player or right panel), so it is only on screen while the + * human is viewing that thread and the app window itself is showing. */ export function resolvePreviewPresentation(input: { readonly visible: boolean; + readonly windowVisible: boolean; readonly shownInThread: boolean; }): NonNullable { - if (input.visible) return "on-screen"; - return input.shownInThread ? "shown-in-thread" : "background"; + if (input.visible && input.windowVisible) return "on-screen"; + return input.visible || input.shownInThread ? "shown-in-thread" : "background"; } diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index e6bedf06d6cfc..940c8e090ec28 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -75,14 +75,14 @@ export const PreviewAutomationStatus = Schema.Struct({ /** Measured guest-page viewport in CSS pixels when a webview is ready. */ viewport: Schema.optional(PreviewRenderedViewportSize), /** - * Why `visible` is what it is. `visible` only reports whether the human can see - * the tab right now, so an `open: true` tab in a thread the human is not looking + * Why `visible` is what it is. `visible` only reports whether the tab is laid + * out in the app's current view, so an `open: true` tab in a thread the human is not looking * at is "shown-in-thread" and not visible. Automation works in every state. */ presentation: Schema.optional( Schema.Literals(["on-screen", "shown-in-thread", "background"]).annotate({ description: - "on-screen: the human can see the tab now. shown-in-thread: it is the thread's open preview, but the human is viewing something else. background: not presented (open=false or closed by the human). Automation works in every state.", + "on-screen: the tab is in the app's current view and the app window is showing. shown-in-thread: it is the thread's open preview (floating or right panel), but the human is viewing something else or the window is hidden. background: not presented (open=false or closed by the human). Automation works in every state.", }), ), });