diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 7b8729a11646c..de116667ecf9a 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -1061,33 +1061,38 @@ describe("DesktopBrowserHost", () => { }).pipe(Effect.scoped), ); - it.effect("tells a download the person clicked from one the agent's input started", () => + it.effect("tells a person's action from one the agent's input started", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const debuggee = makeDebuggee(); host.attach(key, debuggee.tab); + const replies = yield* watchReplies(host); + const raw = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); - yield* host.handleCommandLine( - encodeJson({ - type: "cdp", - ...key, - message: encodeJson({ id: 1, method: "DOM.getDocument", params: {} }), - }), - ); - expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(true); - yield* host.handleCommandLine( - encodeJson({ - type: "cdp", - ...key, - message: encodeJson({ - id: 2, - method: "Input.dispatchMouseEvent", - params: { type: "mousePressed", x: 1, y: 1 }, - }), - }), - ); - expect(host.humanStartedDownload(debuggee.tab.webContents)).toBe(false); - }), + yield* raw({ id: 1, method: "DOM.getDocument", sessionId: "t3-preview-page" }); + expect(host.agentActedRecently(debuggee.tab.webContents)).toBe(false); + yield* raw({ id: 2, method: "Input.dispatchMouseEvent", sessionId: "t3-preview-page" }); + expect(host.agentActedRecently(debuggee.tab.webContents)).toBe(true); + // A person clicking through a viewer that streams the guest, right after the agent. + yield* raw({ id: 3, method: "Target.attachToTarget", params: { targetId: "GUEST" } }); + expect(yield* Queue.take(replies)).toBe(3); + yield* raw({ id: 4, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + yield* raw({ + id: 5, + method: "Input.dispatchMouseEvent", + params: { type: "mouseMoved", x: 1, y: 1 }, + sessionId: "t3-preview-page-1", + }); + expect(host.agentActedRecently(debuggee.tab.webContents)).toBe(true); + yield* raw({ + id: 6, + method: "Input.dispatchMouseEvent", + params: { type: "mousePressed", x: 1, y: 1 }, + sessionId: "t3-preview-page-1", + }); + expect(host.agentActedRecently(debuggee.tab.webContents)).toBe(false); + }).pipe(Effect.scoped), ); it.effect("detaches only the tab whose CDP event is too large to relay", () => diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 2168532902e15..e9b7ca4df8965 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -21,11 +21,8 @@ import { sendPageCommand } from "./GuestInput.ts"; const encodeEvent = Schema.encodeSync(Schema.fromJsonString(DesktopBrowserEvent)); const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBrowserCommand)); const lineEncoder = new TextEncoder(); -const AGENT_INPUT_COMMAND = - /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; -/** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; -const AGENT_DOWNLOAD_WINDOW_MS = 5_000; +const AGENT_INPUT_WINDOW_MS = 5_000; const FOCUS_IDLE = "5 seconds"; /** Well inside the server's 10 s DESKTOP_ATTACH_TIMEOUT, for a first page that never reports it is up. */ const FIRST_PAGE_TIMEOUT = "5 seconds"; @@ -94,7 +91,7 @@ export class DesktopBrowserHost extends Context.Service< readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void; readonly detach: (key: DesktopBrowserTabKey) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; - readonly humanStartedDownload: (source: Electron.WebContents) => boolean; + readonly agentActedRecently: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; readonly pointers: Stream.Stream<{ readonly key: DesktopBrowserTabKey; @@ -327,11 +324,14 @@ export const make = Effect.gen(function* () { tab.screencasts.delete(owner); } // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. - if ( - AGENT_INPUT_METHOD.test(method) && - (owner === undefined || !streamingOwners.has(owner)) - ) { - tab.agentUsed = true; + if (AGENT_INPUT_METHOD.test(method)) { + if (owner === undefined || !streamingOwners.has(owner)) { + tab.agentUsed = true; + tab.agentInputAt = performance.now(); + } else if (params["type"] === "mousePressed" || params["type"] === "touchStart") { + // The viewer's press also hands focus back, so it must end the agent's window. + tab.agentInputAt = Number.NEGATIVE_INFINITY; + } } // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. const forward = (current?: () => boolean) => { @@ -543,13 +543,12 @@ export const make = Effect.gen(function* () { dropRelay(tab); return; } - if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); relayFor(tab).receive(command.value.message); }); - const humanStartedDownload = (source: Electron.WebContents) => { + const agentActedRecently = (source: Electron.WebContents) => { const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); - return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; + return tab !== undefined && performance.now() - tab.agentInputAt <= AGENT_INPUT_WINDOW_MS; }; const announceAll = Effect.suspend(() => { @@ -590,7 +589,7 @@ export const make = Effect.gen(function* () { detach, setShown, placeDownload, - humanStartedDownload, + agentActedRecently, }); }); diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index b4e4b2a8b75ce..3a338f27115fa 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -615,7 +615,7 @@ describe("PreviewManager", () => { attach: () => {}, detach: () => {}, setShown: () => {}, - humanStartedDownload: () => true, + agentActedRecently: () => false, placeDownload: () => true, }), Layer.mock(PreviewPasskeys.PreviewPasskeys)({ @@ -1986,6 +1986,72 @@ describe("PreviewManager", () => { }).pipe(Effect.provide(managerLayer()), Effect.scoped), ); + effectIt.effect("opens a new-tab link the agent clicked in the background", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.DesktopBrowserHost; + const serverTab = { threadId: "thread-1", tabId: "server-tab-1" }; + const contents = Object.assign( + makeTestPreviewWebContents(async () => { + throw new Error("unused"); + }, 42), + { isDevToolsOpened: () => false, getUserAgent: () => "Electron", focusedFrame: {} }, + ); + fromId.mockReturnValue(contents); + const manager = yield* PreviewManager.PreviewManager; + const openLinks = yield* Queue.unbounded<{ url: string; background: boolean }>(); + yield* manager.subscribeOpenLinks(({ url, background }) => + Queue.offer(openLinks, { url, background }).pipe(Effect.asVoid), + ); + const attached = yield* Deferred.make(); + yield* host.events.pipe( + Stream.runForEach((line) => + new TextDecoder().decode(line).includes('"type":"attached"') + ? Deferred.succeed(attached, undefined) + : Effect.void, + ), + Effect.forkScoped, + ); + yield* manager.createTab("tab_1", { serverTab }); + yield* manager.registerWebview("tab_1", 42); + yield* Deferred.await(attached); + const openWindow = vi.mocked(contents.setWindowOpenHandler).mock.calls[0]![0]; + const clickBlankLink = (url: string) => + openWindow({ url, disposition: "foreground-tab" } as Electron.HandlerDetails); + + clickBlankLink("https://example.com/person"); + expect(yield* Queue.take(openLinks)).toEqual({ + url: "https://example.com/person", + background: false, + }); + + Object.assign(contents, { focusedFrame: null }); + yield* host.handleCommandLine( + JSON.stringify({ + type: "cdp", + ...serverTab, + message: JSON.stringify({ + id: 1, + method: "Input.dispatchMouseEvent", + params: { type: "mousePressed", x: 10, y: 10, button: "left", clickCount: 1 }, + sessionId: "t3-preview-page", + }), + }), + ); + clickBlankLink("https://example.com/agent"); + expect(yield* Queue.take(openLinks)).toEqual({ + url: "https://example.com/agent", + background: true, + }); + + Object.assign(contents, { focusedFrame: {} }); + clickBlankLink("https://example.com/person-after-agent"); + expect(yield* Queue.take(openLinks)).toEqual({ + url: "https://example.com/person-after-agent", + background: false, + }); + }).pipe(Effect.provide(managerLayer()), Effect.scoped), + ); + effectIt.effect("discards a screenshot that resolves after its guest is replaced", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index c668c39061e00..524a01dfffb5b 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1565,7 +1565,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function emitOpenLink({ tabId, url: details.url, - background: details.disposition === "background-tab", + // A person's click focuses a guest frame; GuestInput hands focus back after the agent's press. + background: + details.disposition === "background-tab" || + (browserHost.agentActedRecently(wc) && wc.focusedFrame === null), }), ); return { action: "deny" }; @@ -3709,8 +3712,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { downloadSessions.add(session); session.on("will-download", (_event, item, source) => { // Decided before the server's path is set: placing it consumes the guid. - const human = browserHost.humanStartedDownload(source); - if (!browserHost.placeDownload(source, item) || !human) return; + const agentStarted = browserHost.agentActedRecently(source); + if (!browserHost.placeDownload(source, item) || agentStarted) return; item.once("done", (_doneEvent, state) => { if (state !== "completed") return; runForkDownload(