diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 393e4cccccf9c..d6f25d66aa810 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -1298,6 +1298,41 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("stops crediting the agent once a person acts through a viewer", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const debuggee = makeDebuggee(); + host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const viewer = (id: number, method: string, type: string) => + raw({ id, method, params: { type }, sessionId: "t3-preview-page-1" }); + const led = () => host.agentLedPage(debuggee.tab.webContents); + + yield* command(host, 1, "DOM.getDocument"); + expect(led()).toBe(false); + yield* raw({ id: 2, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(2); + yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + + for (const [method, type] of [ + ["Input.dispatchKeyEvent", "rawKeyDown"], + ["Input.dispatchMouseEvent", "mousePressed"], + ["Input.dispatchTouchEvent", "touchStart"], + ] as const) { + yield* command(host, 4, "Runtime.evaluate"); + expect(led()).toBe(true); + // The viewer's pointer only passes over the page the agent drives. + yield* viewer(5, "Input.dispatchMouseEvent", "mouseMoved"); + yield* viewer(6, "Input.dispatchMouseEvent", "mouseWheel"); + expect(led()).toBe(true); + yield* viewer(7, method, type); + expect(led()).toBe(false); + } + }).pipe(Effect.scoped), + ); + it.effect("detaches only the tab whose CDP event is too large to relay", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index e5f43634d204a..296be2fa6709f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -26,7 +26,9 @@ const AGENT_INPUT_COMMAND = /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; /** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; -const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const AGENT_INPUT_WINDOW_MS = 5_000; +/** A viewer's pointer travelling over the stream activates nothing on the page. */ +const POINTER_TRAVEL = new Set(["mouseMoved", "mouseWheel"]); const FOCUS_IDLE = "5 seconds"; /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */ const FIRST_PAGE_TIMEOUT = "5 seconds"; @@ -60,6 +62,8 @@ interface AttachedTab { downloadDirectory: string | null; pendingDownloadGuid: string | null; agentInputAt: number; + /** When the agent last drove the page with no person's input through a viewer since. */ + agentLedAt: number; focusEmulated: boolean; focusChange: Promise | null; focusUses: number; @@ -98,6 +102,8 @@ export class DesktopBrowserHost extends Context.Service< readonly loadFailed: (key: DesktopBrowserTabKey, failure: PreviewAutomationLoadError) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; + /** Whether what the page does now follows the agent's input, not a viewer's. */ + readonly agentLedPage: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; readonly pointers: Stream.Stream<{ readonly key: DesktopBrowserTabKey; @@ -351,11 +357,13 @@ export const make = Effect.gen(function* () { if (tab.screencasts.size > 0) return Promise.resolve({}); } // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. - if ( - AGENT_INPUT_METHOD.test(method) && - (owner === undefined || !streamingOwners.has(owner)) - ) { - tab.agentUsed = true; + if (AGENT_INPUT_METHOD.test(method)) { + if (owner === undefined || !streamingOwners.has(owner)) { + tab.agentUsed = true; + tab.agentLedAt = performance.now(); + } else if (method.startsWith("Input.") && !POINTER_TRAVEL.has(params["type"])) { + tab.agentLedAt = Number.NEGATIVE_INFINITY; + } } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { @@ -490,6 +498,7 @@ export const make = Effect.gen(function* () { downloadDirectory: null, pendingDownloadGuid: null, agentInputAt: Number.NEGATIVE_INFINITY, + agentLedAt: Number.NEGATIVE_INFINITY, focusEmulated: false, focusChange: null, focusUses: 0, @@ -604,7 +613,12 @@ export const make = Effect.gen(function* () { const humanStartedDownload = (source: Electron.WebContents) => { const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); - return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; + return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_INPUT_WINDOW_MS; + }; + + const agentLedPage = (source: Electron.WebContents) => { + const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); + return tab !== undefined && performance.now() - tab.agentLedAt <= AGENT_INPUT_WINDOW_MS; }; const announceAll = Effect.suspend(() => { @@ -647,6 +661,7 @@ export const make = Effect.gen(function* () { setShown, placeDownload, humanStartedDownload, + agentLedPage, }); }); diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 530a1200378b1..e8ce5eaae3696 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -619,6 +619,7 @@ describe("PreviewManager", () => { setShown: () => {}, loadFailed: () => {}, humanStartedDownload: () => true, + agentLedPage: () => false, placeDownload: () => true, }), Layer.mock(PreviewPasskeys.PreviewPasskeys)({ @@ -2196,6 +2197,71 @@ describe("PreviewManager", () => { }).pipe(Effect.provide(managerLayer())), ); + effectIt.effect( + "opens a new-tab link in the background only when the agent's input led to it", + () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { + isDevToolsOpened: () => false, + getUserAgent: () => "Electron", + focusedFrame: null, + }, + ); + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const openLinks = yield* Queue.unbounded(); + yield* manager.subscribeOpenLinks(({ background }) => + Queue.offer(openLinks, background).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + const openWindow = vi.mocked(contents.setWindowOpenHandler).mock.calls[0]![0]; + const openBlankLink = Effect.suspend(() => { + openWindow({ + url: "https://example.com/next", + disposition: "foreground-tab", + } as Electron.HandlerDetails); + return Queue.take(openLinks); + }); + + expect(yield* openBlankLink).toBe(false); + + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Runtime.evaluate", + params: { expression: "document.querySelector('a').click()" }, + sessionId: "t3-preview-page", + }), + }), + ); + expect(yield* openBlankLink).toBe(true); + + // A person clicking in the guest itself gives it a focused frame. + Object.assign(contents, { focusedFrame: {} }); + expect(yield* openBlankLink).toBe(false); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + effectIt.effect("reports early and subsequent load failures for the current guest", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.DesktopBrowserHost; diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 49b61c73316f3..56957783405b1 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1656,7 +1656,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function emitOpenLink({ tabId, url: details.url, - background: details.disposition === "background-tab", + // A person's own click or key leaves the guest a focused frame; GuestInput hands + // focus back after the agent's press, so a foreground tab would take it again. + background: + details.disposition === "background-tab" || + (wc.focusedFrame === null && browserHost.agentLedPage(wc)), }), ); return { action: "deny" };