Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions apps/desktop/src/preview/Manager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3101,6 +3101,91 @@ describe("PreviewManager", () => {
),
);

effectIt.effect(
"lifts window and guest throttling while a still capture runs, without undoing a recording's",
() =>
withManager((manager) =>
Effect.gen(function* () {
// Packaged builds throttle the unfocused main window, and a throttled
// offscreen guest paints too slowly for capturePage's deadline.
let windowThrottled = true;
const windowThrottling = vi.fn((enabled: boolean) => {
windowThrottled = enabled;
});
const throttlingAtCapture: Array<{ window: boolean; guest: boolean }> = [];
const capturePage = vi.fn(async () => {
throttlingAtCapture.push({
window: windowThrottled,
guest: webContents.setBackgroundThrottling.mock.calls.at(-1)?.[0] ?? true,
});
return {
toJPEG: () => Buffer.from("frame"),
toPNG: () => Buffer.from("png"),
getSize: () => ({ width: 1280, height: 720 }),
isEmpty: () => false,
};
});
const host = makeTestHostWebContents();
const webContents = makeTestPreviewWebContents(capturePage, 42, host);
fromId.mockReturnValue(webContents);
yield* manager.createTab("tab_still_capture");
yield* manager.registerWebview("tab_still_capture", 42);
yield* manager.setMainWindow({
isDestroyed: () => false,
once: vi.fn(),
webContents: { setBackgroundThrottling: windowThrottling },
} as never);

yield* manager.captureScreenshot("tab_still_capture");
expect(throttlingAtCapture.at(-1)).toEqual({ window: false, guest: false });
expect(windowThrottled).toBe(true);
expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([true]);

// A recording keeps both unthrottled; a screenshot during it must not re-throttle.
yield* manager.startRecording("tab_still_capture");
yield* manager.captureScreenshot("tab_still_capture");
expect(windowThrottled).toBe(false);
expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([false]);
yield* manager.stopRecording("tab_still_capture");
expect(windowThrottled).toBe(true);
expect(webContents.setBackgroundThrottling.mock.calls.at(-1)).toEqual([true]);
}),
),
);

effectIt.effect("leaves an unthrottled main window unthrottled after a still capture", () =>
withManager((manager) =>
Effect.gen(function* () {
// Dev builds keep the main window unthrottled so input stays responsive.
let windowThrottled = false;
const windowThrottling = vi.fn((enabled: boolean) => {
windowThrottled = enabled;
});
const capturePage = vi.fn(async () => ({
toJPEG: () => Buffer.from("frame"),
toPNG: () => Buffer.from("png"),
getSize: () => ({ width: 1280, height: 720 }),
isEmpty: () => false,
}));
const webContents = makeTestPreviewWebContents(capturePage, 42);
fromId.mockReturnValue(webContents);
yield* manager.createTab("tab_dev_capture");
yield* manager.registerWebview("tab_dev_capture", 42);
yield* manager.setMainWindow({
isDestroyed: () => false,
once: vi.fn(),
webContents: {
setBackgroundThrottling: windowThrottling,
getBackgroundThrottling: () => windowThrottled,
},
} as never);

yield* manager.captureScreenshot("tab_dev_capture");
expect(windowThrottled).toBe(false);
}),
),
);

effectIt.effect("continues native recording when the source warmup fails", () =>
withManager((manager) =>
Effect.gen(function* () {
Expand Down Expand Up @@ -4153,6 +4238,72 @@ describe("PreviewManager", () => {
),
);

effectIt.effect(
"waitFor times out while the selector has no match and resolves once it does",
() =>
withManager((manager) =>
Effect.gen(function* () {
const present = new Set<string>();
// Mirrors Playwright's injected script: querySelector returns undefined on no match.
const pageContext = NodeVM.createContext({
document: { body: { innerText: "" } },
location: { href: "http://127.0.0.1/" },
__t3PlaywrightInjected: {
parseSelector: (selector: string) => selector,
querySelector: (selector: string) => (present.has(selector) ? {} : undefined),
},
});
const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) =>
method === "Runtime.evaluate"
? {
result: {
value: NodeVM.runInContext(String(params?.["expression"]), pageContext),
},
}
: undefined,
);
const webContents = Object.assign(
makeTestPreviewWebContents(async () => {
throw new Error("unused");
}, 52),
{
isDevToolsOpened: () => false,
debugger: {
isAttached: () => false,
attach: vi.fn(),
detach: vi.fn(),
sendCommand,
on: vi.fn(),
off: vi.fn(),
},
},
);
fromId.mockReturnValue(webContents);
yield* manager.createTab("tab_wait_for");
yield* manager.registerWebview("tab_wait_for", 52);

const missing = yield* manager
.automationWaitFor("tab_wait_for", { selector: "#never-appears", timeoutMs: 300 })
.pipe(
Effect.match({ onFailure: (error) => error, onSuccess: () => "matched" as const }),
Effect.forkChild({ startImmediately: true }),
);
yield* TestClock.adjust(500);
expect(yield* Fiber.join(missing)).toMatchObject({
_tag: "PreviewAutomationTimeoutError",
});

const appears = yield* manager
.automationWaitFor("tab_wait_for", { selector: "#later", timeoutMs: 5_000 })
.pipe(Effect.forkChild({ startImmediately: true }));
yield* TestClock.adjust(200);
present.add("css=#later");
yield* TestClock.adjust(200);
yield* Fiber.join(appears);
}),
),
);

effectIt.effect("types in background webviews and enables native key input", () =>
withManager((manager) =>
Effect.gen(function* () {
Expand Down
94 changes: 89 additions & 5 deletions apps/desktop/src/preview/Manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
const frameCaptureSessionsRef = yield* SynchronizedRef.make<
ReadonlyMap<string, FrameCaptureSession>
>(new Map());
// Still captures (snapshots, screenshots) in flight, by guest WebContents id.
// Packaged builds throttle the main window when it is not focused, which slows
// an offscreen guest to about one frame a second, so capturePage misses its
// deadline. Captures lift throttling while they run. Counting them alongside
// frame-capture sessions keeps either one from re-throttling under the other.
const stillCaptureLeasesRef = yield* Ref.make<ReadonlyMap<number, number>>(new Map());
// The main window's throttling before the first lease, restored after the last.
// Dev builds keep the window unthrottled on purpose; packaged builds throttle it.
const stillCaptureWindowThrottlingRef = yield* Ref.make<boolean>(true);
const stillCaptureLeaseTotal = Ref.get(stillCaptureLeasesRef).pipe(
Effect.map((leases) => Array.from(leases.values()).reduce((sum, count) => sum + count, 0)),
);
const pictureInPictureSessionsRef = yield* SynchronizedRef.make<
ReadonlyMap<string, PictureInPictureSession>
>(new Map());
Expand Down Expand Up @@ -736,6 +748,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
schedule: Schedule.spaced(CAPTURE_PAGE_RETRY_DELAY_MS),
while: isPreviewOperationError,
}),
withStillCaptureUnthrottled(wc),
);
});
const currentIso = DateTime.now.pipe(Effect.map(DateTime.formatIso));
Expand Down Expand Up @@ -849,6 +862,70 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
);
});
});
const isHeldByFrameCapture = (
sessions: ReadonlyMap<string, FrameCaptureSession>,
webContentsId: number,
) =>
Array.from(sessions.values()).some((session) =>
session.unthrottledWebContentsIds.has(webContentsId),
);
const acquireStillCaptureUnthrottled = (wc: Electron.WebContents) =>
SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) =>
Effect.gen(function* () {
const leases = yield* Ref.get(stillCaptureLeasesRef);
if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) {
const mainWindow = yield* Ref.get(mainWindowRef);
yield* Ref.set(
stillCaptureWindowThrottlingRef,
Option.isSome(mainWindow) && !mainWindow.value.isDestroyed()
? mainWindow.value.webContents.getBackgroundThrottling?.() !== false
: true,
);
yield* setFrameCaptureBackgroundThrottling(false).pipe(Effect.ignore);
}
if ((leases.get(wc.id) ?? 0) === 0 && !isHeldByFrameCapture(sessions, wc.id)) {
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(Effect.ignore);
}
yield* Ref.set(
stillCaptureLeasesRef,
replaceMap(leases, (copy) => {
copy.set(wc.id, (copy.get(wc.id) ?? 0) + 1);
}),
);
return [undefined, sessions] as const;
}),
);
const releaseStillCaptureUnthrottled = (wc: Electron.WebContents) =>
SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) =>
Effect.gen(function* () {
const leases = yield* Ref.get(stillCaptureLeasesRef);
const remaining = (leases.get(wc.id) ?? 1) - 1;
yield* Ref.set(
stillCaptureLeasesRef,
replaceMap(leases, (copy) => {
if (remaining > 0) copy.set(wc.id, remaining);
else copy.delete(wc.id);
}),
);
if (remaining === 0 && !isHeldByFrameCapture(sessions, wc.id)) {
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, true).pipe(Effect.ignore);
}
if (sessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) {
yield* setFrameCaptureBackgroundThrottling(
yield* Ref.get(stillCaptureWindowThrottlingRef),
).pipe(Effect.ignore);
}
return [undefined, sessions] as const;
}),
).pipe(Effect.uninterruptible);
const withStillCaptureUnthrottled =
(wc: Electron.WebContents) =>
<A, E, R>(capture: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
Effect.acquireUseRelease(
acquireStillCaptureUnthrottled(wc),
() => capture,
() => releaseStillCaptureUnthrottled(wc),
);
const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* (
tabId: string,
consumer: FrameCaptureConsumer,
Expand Down Expand Up @@ -878,10 +955,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
const remainingSessions = replaceMap(sessions, (copy) => {
copy.delete(tabId);
});
const stillCaptureLeases = yield* Ref.get(stillCaptureLeasesRef);
yield* restoreFrameCaptureWebContentsBackgroundThrottling(
current.unthrottledWebContentsIds,
new Set(
Array.from(current.unthrottledWebContentsIds).filter(
(webContentsId) => (stillCaptureLeases.get(webContentsId) ?? 0) === 0,
),
),
);
if (remainingSessions.size === 0) {
if (remainingSessions.size === 0 && (yield* stillCaptureLeaseTotal) === 0) {
yield* setFrameCaptureBackgroundThrottling(true).pipe(
Effect.retry({ times: 2 }),
Effect.catch((error) =>
Expand Down Expand Up @@ -2048,7 +2130,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
}
yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) =>
Effect.gen(function* () {
if (sessions.size > 0) {
if (sessions.size > 0 || (yield* stillCaptureLeaseTotal) > 0) {
yield* setWindowBackgroundThrottling(window, false);
}
yield* Ref.set(mainWindowRef, Option.some(window));
Expand Down Expand Up @@ -3114,9 +3196,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
if (sessions.size === 0) {
yield* setFrameCaptureBackgroundThrottling(false);
}
const stillCapturesActive = (yield* stillCaptureLeaseTotal) > 0;
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
Effect.onError(() =>
sessions.size === 0
sessions.size === 0 && !stillCapturesActive
? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore)
: Effect.void,
),
Expand Down Expand Up @@ -4500,7 +4583,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
send,
`(() => {
try {
const selectorMatched = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, false) !== null; })()` : "true"};
// Playwright's querySelector returns undefined, not null, when nothing matches.
const selectorMatched = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, false) != null; })()` : "true"};
const textMatched = ${
textJson ? `(document.body?.innerText || "").includes(${textJson})` : "true"
};
Expand Down
4 changes: 2 additions & 2 deletions apps/server/src/mcp/toolkits/preview/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ const readonlyBrowserTool = <T extends Tool.Any>(tool: T): T =>

const PreviewStatusTool = Tool.make("preview_status", {
description:
"Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility, loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.",
"Report whether a collaborative browser tab is automation-capable, including its URL, title, visibility (with presentation explaining why it is or is not on screen), loading state, viewport mode, and measured CSS-pixel size. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab.",
parameters: PreviewAutomationTabTargetInput,
success: PreviewAutomationStatus,
failure: PreviewAutomationError,
Expand All @@ -66,7 +66,7 @@ const PreviewStatusTool = Tool.make("preview_status", {
const PreviewOpenTool = browserTool(
Tool.make("preview_open", {
description:
"Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.",
"Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. visible is true only while the human is looking at the tab; presentation says why it is not (shown-in-thread means the preview is open in this thread but the human is viewing something else). Automation works either way. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.",
parameters: PreviewAutomationOpenInput,
success: PreviewAutomationStatus,
failure: PreviewAutomationError,
Expand Down
20 changes: 19 additions & 1 deletion apps/web/src/components/preview/PreviewAutomationHosts.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ import { useAtomQueryRunner } from "~/state/use-atom-query-runner";
import { useAtomCommand } from "~/state/use-atom-command";

import { previewBridge } from "./previewBridge";
import { resolvePreviewPresentation } from "./previewPresentation";
import { selectActiveRightPanelSurface, useRightPanelStore } from "~/rightPanelStore";
import {
PreviewAutomationOperationError,
PreviewAutomationOverlayTimeoutError,
Expand Down Expand Up @@ -237,6 +239,21 @@ const currentStatus = async (
const visible = runtimeTabId
? (useBrowserSurfaceStore.getState().byTabId[runtimeTabId]?.visible ?? false)
: false;
const rightPanelSurface = selectActiveRightPanelSurface(
useRightPanelStore.getState().byThreadKey,
threadRef,
);
const presentation = resolvePreviewPresentation({
visible,
windowVisible: document.visibilityState === "visible",
shownInThread:
tabId !== null &&
(selectThreadPreviewMiniPlayerTabId(
usePreviewMiniPlayerStore.getState().byThreadKey,
threadRef,
) === tabId ||
(rightPanelSurface?.kind === "preview" && rightPanelSurface.resourceId === tabId)),
});
const renderingActive = runtimeTabId ? isPreviewWebviewRendering(runtimeTabId) : false;
const viewportSetting = snapshot ? (snapshot.viewport ?? FILL_PREVIEW_VIEWPORT) : undefined;
const viewport =
Expand All @@ -249,12 +266,13 @@ const currentStatus = async (
};
if (runtimeTabId && tabId && previewBridge && state.desktopByTabId[tabId]) {
const status = await previewBridge.automation.status(runtimeTabId);
return { ...status, tabId, visible, ...viewportStatus };
return { ...status, tabId, visible, presentation, ...viewportStatus };
}
const navStatus = snapshot?.navStatus;
return {
available: false,
visible,
presentation,
tabId,
url: navStatus && navStatus._tag !== "Idle" ? navStatus.url : null,
title: navStatus && navStatus._tag !== "Idle" ? navStatus.title : null,
Expand Down
23 changes: 23 additions & 0 deletions apps/web/src/components/preview/previewPresentation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { describe, expect, it } from "vite-plus/test";

import { resolvePreviewPresentation } from "./previewPresentation";

describe("resolvePreviewPresentation", () => {
it("separates an off-screen thread preview from a background tab", () => {
expect(
resolvePreviewPresentation({ visible: true, windowVisible: true, shownInThread: true }),
).toBe("on-screen");
expect(
resolvePreviewPresentation({ visible: false, windowVisible: true, shownInThread: true }),
).toBe("shown-in-thread");
expect(
resolvePreviewPresentation({ visible: false, windowVisible: true, shownInThread: false }),
).toBe("background");
});

it("does not claim on-screen while the app window is minimized or hidden", () => {
expect(
resolvePreviewPresentation({ visible: true, windowVisible: false, shownInThread: true }),
).toBe("shown-in-thread");
});
});
Loading
Loading