diff --git a/apps/desktop/src/preview/CdpRelay.test.ts b/apps/desktop/src/preview/CdpRelay.test.ts index d409ad8b42e09..09505ffc5babe 100644 --- a/apps/desktop/src/preview/CdpRelay.test.ts +++ b/apps/desktop/src/preview/CdpRelay.test.ts @@ -53,6 +53,7 @@ describe("CDP relay", () => { ); const written: Array> = []; const relay = makeRelay(makeTarget(send), (raw) => written.push(JSON.parse(raw))); + relay.event("Target.attachedToTarget", { sessionId: "iframe-session" }, ""); relay.receive( JSON.stringify({ id: 5, method: "Page.captureScreenshot", sessionId: "t3-preview-page" }), ); @@ -76,6 +77,7 @@ describe("CDP relay", () => { it("resumes paused child frames, so cross-site iframes run", async () => { const send = vi.fn(async () => ({})); const relay = makeRelay(makeTarget(send), () => {}); + relay.event("Target.attachedToTarget", { sessionId: "iframe-session" }, ""); relay.receive( JSON.stringify({ id: 1, @@ -122,6 +124,7 @@ describe("CDP relay", () => { expect(written).toEqual([]); relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); await settle(); + relay.event("Target.attachedToTarget", { sessionId: "iframe-session" }, ""); relay.event("Page.loadEventFired", { timestamp: 1 }, ""); relay.event("Runtime.consoleAPICalled", {}, "iframe-session"); expect(written.slice(-2)).toEqual([ @@ -167,6 +170,397 @@ describe("CDP relay extra sessions", () => { written.length = 0; relay.event("Page.screencastFrame", { data: "x" }, undefined); expect(written.map((m) => m["sessionId"])).toEqual(["t3-preview-page", extra]); + relay.receive( + JSON.stringify({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: extra }, + sessionId: browserSession, + }), + ); + await settle(); + expect(written).toContainEqual({ + method: "Target.detachedFromTarget", + params: { sessionId: extra, targetId: "GUEST-TARGET" }, + sessionId: browserSession, + }); + }); +}); + +describe("CDP relay popups", () => { + const makePopup = ( + targetId: string, + send: CdpRelayTarget["send"], + close: () => void = () => {}, + setDownloadDirectory: CdpRelayTarget["setDownloadDirectory"] = () => {}, + ): CdpRelayTarget => ({ + ...makeTarget(send, setDownloadDirectory), + targetId: async () => targetId, + url: () => `http://localhost:4719/${targetId}`, + close, + }); + const resultOf = (written: Array>, id: number) => + written.find((message) => message["id"] === id)!; + + it.each(["announced", "extra"])( + "revokes popup children when its last %s page session detaches without affecting other targets", + async (last) => { + const pageSend = vi.fn(async () => ({})); + const popupSend = vi.fn(async () => ({})); + const siblingSend = vi.fn(async () => ({})); + const closePopup = vi.fn(); + const written: Array> = []; + const relay = makeRelay(makeTarget(pageSend), (raw) => written.push(JSON.parse(raw))); + const popup = relay.addPopup(makePopup("POPUP", popupSend, closePopup)); + const sibling = relay.addPopup(makePopup("SIBLING", siblingSend)); + relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach" })); + await settle(); + relay.receive( + JSON.stringify({ id: 2, method: "Target.attachToTarget", params: { targetId: "POPUP" } }), + ); + await settle(); + const extra = (resultOf(written, 2)["result"] as { sessionId: string }).sessionId; + const firstSession = last === "announced" ? extra : "t3-preview-popup-1"; + const lastSession = last === "announced" ? "t3-preview-popup-1" : extra; + relay.event("Target.attachedToTarget", { sessionId: "opener-child" }, ""); + sibling.event("Target.attachedToTarget", { sessionId: "sibling-child" }, ""); + popup.event("Target.attachedToTarget", { sessionId: "popup-child" }, ""); + popup.event("Target.attachedToTarget", { sessionId: "nested-child" }, "popup-child"); + relay.receive( + JSON.stringify({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: firstSession }, + }), + ); + await settle(); + relay.receive(JSON.stringify({ id: 4, method: "DOM.enable", sessionId: "popup-child" })); + await settle(); + expect(resultOf(written, 4)).toHaveProperty("result"); + expect(popupSend).toHaveBeenCalledExactlyOnceWith("DOM.enable", {}, "popup-child", undefined); + const childEvent = { + method: "Runtime.consoleAPICalled", + params: {}, + sessionId: "popup-child", + }; + popup.event(childEvent.method, childEvent.params, childEvent.sessionId); + expect(written.at(-1)).toEqual(childEvent); + popupSend.mockClear(); + + relay.receive( + JSON.stringify({ + id: 5, + method: "Target.detachFromTarget", + params: { sessionId: lastSession }, + }), + ); + relay.receive( + JSON.stringify({ + id: 6, + method: "Target.detachFromTarget", + params: { sessionId: "t3-preview-page" }, + }), + ); + await settle(); + for (const sessionId of [firstSession, lastSession]) { + expect(written).toContainEqual({ + method: "Target.detachedFromTarget", + params: { sessionId, targetId: "POPUP" }, + }); + } + const eventCount = written.length; + popup.event(childEvent.method, childEvent.params, childEvent.sessionId); + popup.event("Runtime.consoleAPICalled", {}, "nested-child"); + popup.event("Target.attachedToTarget", { sessionId: "late-child" }, ""); + popup.event("Target.attachedToTarget", { sessionId: "late-nested-child" }, "popup-child"); + popup.event("Page.loadEventFired", {}, ""); + expect(written).toHaveLength(eventCount); + const detached = [ + lastSession, + "popup-child", + "nested-child", + "late-child", + "late-nested-child", + ]; + detached.forEach((sessionId, index) => + relay.receive(JSON.stringify({ id: 10 + index, method: "Runtime.evaluate", sessionId })), + ); + for (const [id, sessionId] of [ + [20, "t3-preview-page"], + [21, "opener-child"], + [22, "sibling-child"], + ]) + relay.receive(JSON.stringify({ id, method: "Page.enable", sessionId })); + relay.receive(JSON.stringify({ id: 23, method: "Target.getTargets" })); + await settle(); + detached.forEach((_, index) => expect(resultOf(written, 10 + index)).toHaveProperty("error")); + expect(popupSend).not.toHaveBeenCalled(); + expect(closePopup).not.toHaveBeenCalled(); + expect(pageSend.mock.calls).toEqual([ + ["Page.enable", {}, undefined, undefined], + ["Page.enable", {}, "opener-child", undefined], + ]); + expect(siblingSend).toHaveBeenCalledExactlyOnceWith( + "Page.enable", + {}, + "sibling-child", + undefined, + ); + for (const id of [20, 21, 22]) expect(resultOf(written, id)).toHaveProperty("result"); + expect(resultOf(written, 23)["result"]).toMatchObject({ + targetInfos: [ + { targetId: "GUEST-TARGET", attached: true }, + { targetId: "POPUP", attached: false }, + { targetId: "SIBLING", attached: true }, + ], + }); + popup.remove(); + expect( + written.filter((message) => message["method"] === "Target.detachedFromTarget"), + ).toHaveLength(2); + }, + ); + + it.each(["browser", "popup", "child"])( + "detaches a popup child through its %s session without affecting the other sessions", + async (source) => { + const pageSend = vi.fn(async () => ({})); + const popupSend = vi.fn(async () => ({})); + const written: Array> = []; + const relay = makeRelay(makeTarget(pageSend), (raw) => written.push(JSON.parse(raw))); + const popup = relay.addPopup(makePopup("POPUP", popupSend)); + relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach" })); + await settle(); + const popupSession = "t3-preview-popup-1"; + popup.event("Target.attachedToTarget", { sessionId: "parent-frame" }, ""); + popup.event("Target.attachedToTarget", { sessionId: "popup-child" }, "parent-frame"); + popup.event("Target.attachedToTarget", { sessionId: "nested-child" }, "popup-child"); + popup.event("Target.attachedToTarget", { sessionId: "sibling-frame" }, ""); + relay.receive(JSON.stringify({ id: 2, method: "DOM.enable", sessionId: "popup-child" })); + await settle(); + expect(popupSend).toHaveBeenLastCalledWith("DOM.enable", {}, "popup-child", undefined); + + const sessionId = + source === "browser" ? undefined : source === "popup" ? popupSession : "parent-frame"; + relay.receive( + JSON.stringify({ + id: 3, + method: "Target.detachFromTarget", + params: { sessionId: "popup-child" }, + sessionId, + }), + ); + await settle(); + expect(popupSend).toHaveBeenLastCalledWith( + "Target.detachFromTarget", + { sessionId: "popup-child" }, + "parent-frame", + undefined, + ); + expect(resultOf(written, 3)).toHaveProperty("result"); + + popupSend.mockClear(); + relay.receive(JSON.stringify({ id: 4, method: "DOM.enable", sessionId: "popup-child" })); + relay.receive(JSON.stringify({ id: 5, method: "Page.enable", sessionId: popupSession })); + relay.receive(JSON.stringify({ id: 6, method: "DOM.enable", sessionId: "sibling-frame" })); + relay.receive(JSON.stringify({ id: 7, method: "Page.enable", sessionId: "t3-preview-page" })); + relay.receive(JSON.stringify({ id: 8, method: "DOM.enable", sessionId: "nested-child" })); + await settle(); + expect(resultOf(written, 4)).toHaveProperty("error"); + expect(resultOf(written, 8)).toHaveProperty("error"); + const eventCount = written.length; + popup.event("Runtime.consoleAPICalled", {}, "popup-child"); + popup.event("Runtime.consoleAPICalled", {}, "nested-child"); + popup.event("Target.attachedToTarget", { sessionId: "late-child" }, "popup-child"); + expect(written).toHaveLength(eventCount); + expect(popupSend.mock.calls).toEqual([ + ["Page.enable", {}, undefined, popupSession], + ["DOM.enable", {}, "sibling-frame", undefined], + ]); + expect(pageSend.mock.calls).toEqual([["Page.enable", {}, undefined, undefined]]); + for (const id of [5, 6, 7]) expect(resultOf(written, id)).toHaveProperty("result"); + }, + ); + + it("announces the page's popups after it, as its popups, and routes them separately", async () => { + const pageSend = vi.fn(async () => ({})); + const earlySend = vi.fn(async () => ({})); + const lateSend = vi.fn(async () => ({})); + const closeLate = vi.fn(); + const directories: Array = []; + const written: Array> = []; + const relay = makeRelay(makeTarget(pageSend), (raw) => written.push(JSON.parse(raw))); + const early = relay.addPopup(makePopup("EARLY", earlySend)); + relay.receive( + JSON.stringify({ + id: 1, + method: "Browser.setDownloadBehavior", + params: { behavior: "allowAndName", downloadPath: "/srv/artifacts", eventsEnabled: true }, + }), + ); + relay.receive(JSON.stringify({ id: 2, method: "Target.setAutoAttach", params: {} })); + await settle(); + const late = relay.addPopup( + makePopup("LATE", lateSend, closeLate, (directory) => directories.push(directory)), + ); + await settle(); + + const announced = written.filter((m) => m["method"] === "Target.attachedToTarget"); + expect(announced.map((m) => m["params"])).toEqual([ + expect.objectContaining({ + sessionId: "t3-preview-page", + targetInfo: expect.not.objectContaining({ openerId: expect.anything() }), + }), + expect.objectContaining({ + targetInfo: expect.objectContaining({ targetId: "EARLY", openerId: "GUEST-TARGET" }), + }), + expect.objectContaining({ + targetInfo: expect.objectContaining({ targetId: "LATE", openerId: "GUEST-TARGET" }), + }), + ]); + const [, earlySession, lateSession] = announced.map( + (m) => (m["params"] as { sessionId: string }).sessionId, + ); + // A popup opened after the server chose where downloads go saves there too. + expect(directories).toEqual(["/srv/artifacts"]); + expect(lateSend).toHaveBeenCalledWith( + "Page.setDownloadBehavior", + { behavior: "default" }, + undefined, + ); + + relay.receive(JSON.stringify({ id: 3, method: "Target.getTargets" })); + relay.receive(JSON.stringify({ id: 4, method: "Target.attachToBrowserTarget" })); + await settle(); + expect( + (resultOf(written, 3)["result"] as { targetInfos: Array<{ targetId: string }> }).targetInfos, + ).toEqual([ + expect.objectContaining({ targetId: "GUEST-TARGET" }), + expect.objectContaining({ targetId: "EARLY" }), + expect.objectContaining({ targetId: "LATE" }), + ]); + const browserSession = (resultOf(written, 4)["result"] as { sessionId: string }).sessionId; + const closedDialog = { result: true, userInput: "" }; + late.event("Page.javascriptDialogOpening", { type: "alert", message: "hello" }, ""); + late.event("Page.javascriptDialogClosed", closedDialog, ""); + relay.receive( + JSON.stringify({ + id: 5, + method: "Target.attachToTarget", + params: { targetId: "LATE", flatten: true }, + sessionId: browserSession, + }), + ); + relay.receive( + JSON.stringify({ id: 6, method: "Target.closeTarget", params: { targetId: "GUEST-TARGET" } }), + ); + relay.receive( + JSON.stringify({ id: 7, method: "Target.attachToTarget", params: { targetId: "OTHER" } }), + ); + await settle(); + const extra = (resultOf(written, 5)["result"] as { sessionId: string }).sessionId; + expect(resultOf(written, 6)).toHaveProperty("error"); + expect(resultOf(written, 7)).toHaveProperty("error"); + + relay.receive(JSON.stringify({ id: 8, method: "Page.enable", sessionId: earlySession })); + relay.receive(JSON.stringify({ id: 9, method: "Page.startScreencast", sessionId: extra })); + relay.receive(JSON.stringify({ id: 10, method: "Page.enable", sessionId: "t3-preview-page" })); + await settle(); + expect(earlySend.mock.calls.at(-1)).toEqual(["Page.enable", {}, undefined, earlySession]); + expect(lateSend.mock.calls.at(-1)).toEqual(["Page.startScreencast", {}, undefined, extra]); + expect(pageSend.mock.calls.at(-1)).toEqual(["Page.enable", {}, undefined, undefined]); + + // The native response predates this CDP session, while popup adoption is waiting. + written.length = 0; + relay.receive(JSON.stringify({ id: 14, method: "Page.enable", sessionId: extra })); + await settle(); + expect(written).toEqual([ + { method: "Page.javascriptDialogClosed", params: closedDialog, sessionId: extra }, + { id: 14, result: {}, sessionId: extra }, + ]); + + const enabling = Promise.withResolvers(); + lateSend.mockImplementationOnce(() => enabling.promise); + written.length = 0; + relay.receive(JSON.stringify({ id: 15, method: "Page.enable", sessionId: extra })); + late.event("Page.javascriptDialogOpening", { type: "confirm", message: "still open" }, ""); + enabling.resolve({}); + await settle(); + expect( + written.filter((message) => message["method"] === "Page.javascriptDialogClosed"), + ).toEqual([]); + expect(resultOf(written, 15)).toMatchObject({ result: {} }); + + // A frame's session belongs to the popup whose debugger attached it. + late.event( + "Target.attachedToTarget", + { sessionId: "late-frame", targetInfo: { type: "iframe" } }, + "", + ); + late.event("Page.javascriptDialogClosed", closedDialog, ""); + late.event("Page.javascriptDialogOpening", { type: "alert", message: "iframe" }, "late-frame"); + written.length = 0; + relay.receive(JSON.stringify({ id: 16, method: "Page.enable", sessionId: extra })); + await settle(); + expect(written).toEqual([{ id: 16, result: {}, sessionId: extra }]); + const closedFrameDialog = { result: false, userInput: "" }; + late.event("Page.javascriptDialogClosed", closedFrameDialog, "late-frame"); + written.length = 0; + relay.receive(JSON.stringify({ id: 17, method: "Page.enable", sessionId: extra })); + await settle(); + expect(written).toEqual([ + { method: "Page.javascriptDialogClosed", params: closedFrameDialog, sessionId: extra }, + { id: 17, result: {}, sessionId: extra }, + ]); + relay.receive(JSON.stringify({ id: 11, method: "DOM.enable", sessionId: "late-frame" })); + await settle(); + expect(lateSend.mock.calls.at(-1)).toEqual(["DOM.enable", {}, "late-frame", undefined]); + + written.length = 0; + early.event("Page.loadEventFired", { timestamp: 1 }, ""); + late.event("Page.downloadWillBegin", { guid: "g1", frameId: "LATE" }, ""); + expect(written).toEqual([ + { method: "Page.loadEventFired", params: { timestamp: 1 }, sessionId: earlySession }, + { method: "Browser.downloadWillBegin", params: { guid: "g1", frameId: "LATE" } }, + ]); + + relay.receive( + JSON.stringify({ id: 12, method: "Target.closeTarget", params: { targetId: "LATE" } }), + ); + await settle(); + expect(closeLate).toHaveBeenCalledOnce(); + written.length = 0; + late.remove(); + late.event("Page.loadEventFired", {}, ""); + expect(written).toEqual([ + { + method: "Target.detachedFromTarget", + params: { sessionId: lateSession, targetId: "LATE" }, + }, + { + method: "Target.detachedFromTarget", + params: { sessionId: extra, targetId: "LATE" }, + sessionId: browserSession, + }, + ]); + relay.receive(JSON.stringify({ id: 13, method: "Target.getTargets" })); + await settle(); + expect( + (resultOf(written, 13)["result"] as { targetInfos: Array<{ targetId: string }> }).targetInfos, + ).toHaveLength(2); + }); + + it("never announces a popup that closed before the page was", async () => { + const written: Array> = []; + const relay = makeRelay(makeTarget(vi.fn()), (raw) => written.push(JSON.parse(raw))); + relay.addPopup(makePopup("GONE", vi.fn())).remove(); + relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); + await settle(); + expect(written.filter((m) => m["method"] !== undefined)).toEqual([ + expect.objectContaining({ + params: expect.objectContaining({ sessionId: "t3-preview-page" }), + }), + ]); }); }); @@ -250,23 +644,35 @@ describe("CDP relay downloads", () => { expect(written).toEqual([]); }); - it.each([false, true])( - "keeps the previous download configuration (events enabled: %s) when configuration fails", - async (enabled) => { - const send = vi - .fn(async () => ({})) - .mockResolvedValueOnce({}) - .mockRejectedValueOnce(new Error("Page command rejected")); + it.each( + [false, true].flatMap((enabled) => + ["page", "popup", "removed popup"].map((failure) => ({ enabled, failure })), + ), + )( + "keeps download configuration consistent when $failure rejects (events enabled: $enabled)", + async ({ enabled, failure }) => { + const send = vi.fn(async () => ({})); + const popupSend = vi.fn(async () => ({})); const written: Array> = []; let directory: string | null = null; + let popupDirectory: string | null = null; const relay = makeRelay( makeTarget(send, (value) => { directory = value; }), (raw) => written.push(JSON.parse(raw)), ); + const popup = relay.addPopup({ + ...makeTarget(popupSend, (value) => { + popupDirectory = value; + }), + targetId: async () => "POPUP", + }); relay.receive(JSON.stringify({ id: 1, method: "Target.setAutoAttach", params: {} })); for (const [id, eventsEnabled] of [enabled, !enabled].entries()) { + const pending = Promise.withResolvers(); + if (id === 1) + (failure === "page" ? send : popupSend).mockImplementationOnce(() => pending.promise); relay.receive( JSON.stringify({ id: id + 2, @@ -278,18 +684,42 @@ describe("CDP relay downloads", () => { }, }), ); + if (id === 1) { + if (failure === "removed popup") popup.remove(); + pending.reject(new Error("Page command rejected")); + } await settle(); } - expect(written).toContainEqual({ - id: 3, - error: { code: -32000, message: "Page command rejected" }, - }); - expect(directory).toBe("/srv/artifacts"); + const removed = failure === "removed popup"; + expect(written).toContainEqual( + removed + ? { id: 3, result: {} } + : { id: 3, error: { code: -32000, message: "Page command rejected" } }, + ); + const expectedDirectory = removed + ? enabled + ? null + : "/srv/new-artifacts" + : "/srv/artifacts"; + expect(directory).toBe(expectedDirectory); + expect(popupDirectory).toBe("/srv/artifacts"); + if (removed) { + const lateSend = vi.fn(async () => ({})); + const lateDirectory = vi.fn(); + relay.addPopup(makeTarget(lateSend, lateDirectory)); + await settle(); + expect(lateDirectory).toHaveBeenCalledExactlyOnceWith(expectedDirectory); + expect(lateSend).toHaveBeenCalledExactlyOnceWith( + "Page.setDownloadBehavior", + { behavior: enabled ? "deny" : "default" }, + undefined, + ); + } written.length = 0; relay.event("Page.downloadWillBegin", { guid: "g1" }, ""); relay.event("Page.downloadProgress", { guid: "g1", state: "completed" }, ""); expect(written).toEqual( - enabled + (removed ? !enabled : enabled) ? [ { method: "Browser.downloadWillBegin", params: { guid: "g1" } }, { method: "Browser.downloadProgress", params: { guid: "g1", state: "completed" } }, diff --git a/apps/desktop/src/preview/CdpRelay.ts b/apps/desktop/src/preview/CdpRelay.ts index 9a8e4e4c68346..50c076cb1c98e 100644 --- a/apps/desktop/src/preview/CdpRelay.ts +++ b/apps/desktop/src/preview/CdpRelay.ts @@ -16,11 +16,23 @@ export interface CdpRelayTarget { readonly userAgent: () => string; /** The desktop and its server share a disk; download paths need no translation. */ readonly setDownloadDirectory: (directory: string | null) => void; + /** Only a popup closes this way; the tab's own page closes with its preview tab. */ + readonly close?: () => void; +} + +type CdpRelayEvent = (method: string, params: unknown, sessionId: string | undefined) => void; + +export interface CdpRelayPopup { + readonly event: CdpRelayEvent; + /** Its window closed, or its debugger went away. */ + readonly remove: () => void; } export interface CdpRelayConnection { readonly receive: (raw: string) => void; - readonly event: (method: string, params: unknown, sessionId: string | undefined) => void; + readonly event: CdpRelayEvent; + /** A window the page opened; Playwright sees it as the page's popup. */ + readonly addPopup: (target: CdpRelayTarget) => CdpRelayPopup; } interface CdpCommand { @@ -70,6 +82,21 @@ const ACKNOWLEDGED = new Set([ "Browser.cancelDownload", ]); +interface RelayedTarget { + readonly target: CdpRelayTarget; + /** The session it is announced on. */ + readonly sessionId: string; + /** Known once announced; a closed target cannot be asked for it. */ + targetId: string | null; + removed: boolean; + closedDialog?: unknown; +} + +type RelaySession = + | { readonly kind: "browser" } + /** Relay-created page sessions share one real debugger; `parent` is the session that attached it. */ + | { readonly kind: "page"; readonly owner: RelayedTarget; readonly parent: string | undefined }; + export function createCdpRelayConnection( target: CdpRelayTarget, write: (message: string) => void, @@ -97,25 +124,86 @@ export function createCdpRelayConnection( closed = true; close(); }; - let attached = false; + let autoAttached = false; let downloadEventsEnabled = false; - /** Relay-created page sessions share one real debugger; child frame sessions retain their own ids. */ - const sessions = new Map([[PAGE_SESSION_ID, "page"]]); + let downloads: { + readonly behavior: "deny" | "default"; + readonly directory: string | null; + } | null = null; + const page: RelayedTarget = { + target, + sessionId: PAGE_SESSION_ID, + targetId: null, + removed: false, + }; + /** The page first, then its popups in the order they opened. */ + const targets = new Set([page]); + const sessions = new Map([ + [PAGE_SESSION_ID, { kind: "page", owner: page, parent: undefined }], + ]); + const children = new Map< + string, + { readonly owner: RelayedTarget; readonly parent: string | undefined } + >(); + const removeChild = (sessionId: string) => { + children.delete(sessionId); + for (const [id, child] of children) if (child.parent === sessionId) removeChild(id); + }; let sessionSequence = 0; - const newSession = (kind: "browser" | "page") => { - const sessionId = `t3-preview-${kind}-${++sessionSequence}`; - sessions.set(sessionId, kind); + const newSession = (session: RelaySession) => { + const sessionId = `t3-preview-${session.kind}-${++sessionSequence}`; + sessions.set(sessionId, session); return sessionId; }; - const targetInfo = async () => ({ - targetId: await target.targetId(), + const announced = (relayed: RelayedTarget) => relayed.targetId !== null && !relayed.removed; + const hasPageSession = (relayed: RelayedTarget) => { + for (const session of sessions.values()) + if (session.kind === "page" && session.owner === relayed) return true; + return false; + }; + const targetInfo = async (relayed: RelayedTarget) => ({ + targetId: await relayed.target.targetId(), type: "page", - title: target.title(), - url: target.url(), - attached: true, - canAccessOpener: false, + title: relayed.target.title(), + url: relayed.target.url(), + attached: hasPageSession(relayed), + ...(relayed === page + ? { canAccessOpener: false } + : { canAccessOpener: true, openerId: await target.targetId() }), browserContextId: BROWSER_CONTEXT_ID, }); + const attachedEvent = (relayed: RelayedTarget, info: { readonly targetId: string }) => { + relayed.targetId = info.targetId; + sessions.set(relayed.sessionId, { kind: "page", owner: relayed, parent: undefined }); + return { + method: "Target.attachedToTarget", + params: { + sessionId: relayed.sessionId, + targetInfo: { ...info, attached: true }, + waitingForDebugger: false, + }, + }; + }; + /** Popups follow the page, one at a time, so Playwright already knows their opener. */ + let announcing = Promise.resolve(); + const announcePopup = (relayed: RelayedTarget) => { + announcing = announcing.then(() => + targetInfo(relayed).then( + (info) => { + if (!relayed.removed && relayed.targetId === null) send(attachedEvent(relayed, info)); + }, + () => {}, + ), + ); + }; + const findTarget = async (targetId: unknown) => { + for (const relayed of targets) { + if (relayed === page || announced(relayed)) { + if ((await relayed.target.targetId()) === targetId) return relayed; + } + } + return undefined; + }; const browserCommand = async (command: CdpCommand): Promise => { if (ACKNOWLEDGED.has(command.method)) return {}; @@ -123,13 +211,19 @@ export function createCdpRelayConnection( case "Browser.setDownloadBehavior": { const directory = command.params?.["downloadPath"]; const allowed = command.params?.["behavior"] !== "deny" && typeof directory === "string"; + const behavior = command.params?.["behavior"] === "deny" ? "deny" : "default"; // Browser-scoped behavior targets Electron's app session; preview paths belong to will-download. - const result = await target.send( - "Page.setDownloadBehavior", - { behavior: command.params?.["behavior"] === "deny" ? "deny" : "default" }, - undefined, + const [result] = await Promise.all( + [...targets].map((relayed) => + relayed.target + .send("Page.setDownloadBehavior", { behavior }, undefined) + .catch((cause: unknown) => { + if (relayed === page || !relayed.removed) throw cause; + }), + ), ); - target.setDownloadDirectory(allowed ? directory : null); + downloads = { behavior, directory: allowed ? directory : null }; + for (const relayed of targets) relayed.target.setDownloadDirectory(downloads.directory); downloadEventsEnabled = command.params?.["eventsEnabled"] === true; return result; } @@ -155,33 +249,61 @@ export function createCdpRelayConnection( case "Target.getBrowserContexts": return { browserContextIds: [] }; case "Target.getTargets": - return { targetInfos: [await targetInfo()] }; + return { + targetInfos: await Promise.all( + [...targets] + .filter((relayed) => relayed === page || announced(relayed)) + .map(targetInfo), + ), + }; case "Target.attachToBrowserTarget": - return { sessionId: newSession("browser") }; + return { sessionId: newSession({ kind: "browser" }) }; case "Target.attachToTarget": { - const targetId = command.params?.["targetId"]; - if (targetId !== (await target.targetId())) - throw new Error("Only this preview tab's page can be attached."); - return { sessionId: newSession("page") }; + const owner = await findTarget(command.params?.["targetId"]); + if (!owner) throw new Error("Only this preview tab's page and its popups can be attached."); + return { sessionId: newSession({ kind: "page", owner, parent: command.sessionId }) }; } case "Target.detachFromTarget": { const sessionId = command.params?.["sessionId"]; - if (typeof sessionId === "string" && sessionId !== PAGE_SESSION_ID) { - sessions.delete(sessionId); - target.sessionDetached?.(sessionId); + const session = typeof sessionId === "string" ? sessions.get(sessionId) : undefined; + const child = typeof sessionId === "string" ? children.get(sessionId) : undefined; + if (child !== undefined) { + await child.owner.target.send( + command.method, + command.params ?? {}, + child.parent, + undefined, + ); + removeChild(sessionId as string); + } else if (session?.kind === "page" && sessionId !== PAGE_SESSION_ID) { + const targetId = session.owner.targetId ?? (await session.owner.target.targetId()); + sessions.delete(sessionId as string); + if (!hasPageSession(session.owner)) + for (const [id, child] of children) + if (child.owner === session.owner) children.delete(id); + session.owner.target.sessionDetached?.(sessionId as string); + send({ + method: "Target.detachedFromTarget", + params: { sessionId, targetId }, + ...(session.parent === undefined ? {} : { sessionId: session.parent }), + }); + } else if (session?.kind === "browser") { + sessions.delete(sessionId as string); } return {}; } + case "Target.closeTarget": { + const owner = await findTarget(command.params?.["targetId"]); + if (owner === undefined || owner === page || !owner.target.close) + throw new Error(`Not supported for a desktop preview tab: ${command.method}`); + owner.target.close(); + return { success: true }; + } case "Target.setAutoAttach": { - if (attached) return {}; - attached = true; - const info = await targetInfo(); - return { - [ANNOUNCE]: { - method: "Target.attachedToTarget", - params: { sessionId: PAGE_SESSION_ID, targetInfo: info, waitingForDebugger: false }, - }, - }; + if (autoAttached) return {}; + autoAttached = true; + const info = await targetInfo(page); + return { [ANNOUNCE]: info }; } default: throw new Error(`Not supported for a desktop preview tab: ${command.method}`); @@ -189,16 +311,89 @@ export function createCdpRelayConnection( }; const pageCommand = (command: CdpCommand): Promise => { - const child = sessions.get(command.sessionId!) === "page" ? undefined : command.sessionId; - const owner = - child === undefined && command.sessionId !== PAGE_SESSION_ID ? command.sessionId : undefined; - // Child frames/workers pause on auto-attach until resumed; only the main page can skip this command. + const session = sessions.get(command.sessionId!); + const child = session?.kind === "page" ? undefined : command.sessionId; + const owner = session?.kind === "page" ? session.owner : children.get(child!)?.owner; + if (owner === undefined) return Promise.reject(new Error("Unknown or detached CDP session")); + if (command.method === "Target.detachFromTarget") return browserCommand(command); + const extra = child === undefined && command.sessionId !== PAGE_SESSION_ID; + // Child frames/workers pause on auto-attach until resumed; only a main page can skip this command. if (command.method === "Runtime.runIfWaitingForDebugger" && child === undefined) { return Promise.resolve({}); } // Playwright asks for this on every page; the host emulates focus only while it is in use. if (command.method === "Emulation.setFocusEmulationEnabled") return Promise.resolve({}); - return target.send(command.method, command.params ?? {}, child, owner); + const result = owner.target.send( + command.method, + command.params ?? {}, + child, + extra ? command.sessionId : undefined, + ); + if (command.method !== "Page.enable" || child !== undefined) return result; + return result.then((result) => { + // Popup adoption may have buffered the dialog before this session existed. + if ( + !owner.removed && + sessions.get(command.sessionId!) === session && + owner.closedDialog !== undefined + ) { + send({ + method: "Page.javascriptDialogClosed", + params: owner.closedDialog, + sessionId: command.sessionId, + }); + } + return result; + }); + }; + + const eventFor = + (relayed: RelayedTarget): CdpRelayEvent => + (method, params, sessionId) => { + if (relayed.removed) return; + if (sessionId && children.get(sessionId)?.owner !== relayed) return; + if (method === "Page.javascriptDialogOpening") relayed.closedDialog = undefined; + else if (method === "Page.javascriptDialogClosed") relayed.closedDialog = params; + if (announced(relayed) && !hasPageSession(relayed)) return; + const child = (params as { sessionId?: unknown } | null | undefined)?.sessionId; + if (method === "Target.attachedToTarget" && typeof child === "string") { + children.set(child, { owner: relayed, parent: sessionId || undefined }); + } else if (method === "Target.detachedFromTarget" && typeof child === "string") { + removeChild(child); + } + if (!announced(relayed)) return; + if (method.startsWith("Page.download")) method = method.replace("Page.", "Browser."); + // Download events are the browser's; Playwright listens on its root session. + if (method.startsWith("Browser.download")) { + if (downloadEventsEnabled) send({ method, params }); + return; + } + // Electron reports the page's own events with an empty session id. + if (sessionId) { + send({ method, params, sessionId }); + return; + } + for (const [id, session] of sessions) + if (session.kind === "page" && session.owner === relayed) + send({ method, params, sessionId: id }); + }; + + const removePopup = (relayed: RelayedTarget) => { + if (relayed.removed) return; + relayed.removed = true; + targets.delete(relayed); + for (const [id, child] of children) if (child.owner === relayed) children.delete(id); + const targetId = relayed.targetId; + if (targetId === null) return; + for (const [id, session] of sessions) { + if (session.kind !== "page" || session.owner !== relayed) continue; + sessions.delete(id); + send({ + method: "Target.detachedFromTarget", + params: { sessionId: id, targetId }, + ...(session.parent === undefined ? {} : { sessionId: session.parent }), + }); + } }; return { @@ -212,19 +407,21 @@ export function createCdpRelayConnection( } if (!isCommand(command)) return; const result = - command.sessionId === undefined || sessions.get(command.sessionId) === "browser" + command.sessionId === undefined || sessions.get(command.sessionId)?.kind === "browser" ? browserCommand(command) : pageCommand(command); const route = command.sessionId === undefined ? {} : { sessionId: command.sessionId }; // Reply as commands finish so slow commands cannot block later replies. void result.then( (value) => { - const announce = + const info = typeof value === "object" && value !== null && ANNOUNCE in value - ? (value as { [ANNOUNCE]: Record })[ANNOUNCE] + ? (value as { [ANNOUNCE]: { readonly targetId: string } })[ANNOUNCE] : undefined; - send({ id: command.id, result: announce ? {} : (value ?? {}), ...route }); - if (announce) send(announce); + send({ id: command.id, result: info ? {} : (value ?? {}), ...route }); + if (!info) return; + send(attachedEvent(page, info)); + for (const relayed of targets) if (relayed !== page) announcePopup(relayed); }, (cause: unknown) => send({ @@ -237,21 +434,23 @@ export function createCdpRelayConnection( }), ); }, - event: (method, params, sessionId) => { - if (!attached) return; - if (method.startsWith("Page.download")) method = method.replace("Page.", "Browser."); - // Download events are the browser's; Playwright listens on its root session. - if (method.startsWith("Browser.download")) { - if (downloadEventsEnabled) send({ method, params }); - return; - } - // Electron reports the page's own events with an empty session id. - if (sessionId) { - send({ method, params, sessionId }); - return; + event: eventFor(page), + addPopup: (popupTarget) => { + const relayed: RelayedTarget = { + target: popupTarget, + sessionId: `t3-preview-popup-${++sessionSequence}`, + targetId: null, + removed: false, + }; + targets.add(relayed); + if (downloads !== null) { + popupTarget.setDownloadDirectory(downloads.directory); + void popupTarget + .send("Page.setDownloadBehavior", { behavior: downloads.behavior }, undefined) + .catch(() => {}); } - for (const [id, kind] of sessions) - if (kind === "page") send({ method, params, sessionId: id }); + if (announced(page)) announcePopup(relayed); + return { event: eventFor(relayed), remove: () => removePopup(relayed) }; }, }; } diff --git a/apps/desktop/src/preview/DesktopBrowserHost.test.ts b/apps/desktop/src/preview/DesktopBrowserHost.test.ts index 7b8729a11646c..d84286eb9fc4f 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.test.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.test.ts @@ -14,7 +14,7 @@ import * as DesktopBrowserHost from "./DesktopBrowserHost.ts"; const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(DesktopBrowserEvent)); const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); const decodeCdpReply = Schema.decodeUnknownSync( - Schema.fromJsonString(Schema.Struct({ id: Schema.Number })), + Schema.fromJsonString(Schema.Struct({ id: Schema.optionalKey(Schema.Number) })), ); const key = { threadId: "thread-1", tabId: "tab-1" }; /** Sends a page command whose id the fake debugger records as it receives it. */ @@ -33,6 +33,7 @@ const command = ( const makeDebuggee = ( options: { + readonly targetId?: string; readonly deferFocus?: boolean; /** The guest is still loading its first page. */ readonly loading?: boolean; @@ -57,7 +58,7 @@ const makeDebuggee = ( return; } if (method === "Target.getTargetInfo") { - resolve({ targetInfo: { targetId: "GUEST" } }); + resolve({ targetInfo: { targetId: options.targetId ?? "GUEST" } }); return; } if (method === "Emulation.setFocusEmulationEnabled") { @@ -70,14 +71,27 @@ const makeDebuggee = ( pending.push(() => resolve({ method })); sendWaiters.shift()?.(); }), + attached: false, + isAttached: () => debuggee.attached, + attach: () => { + if (devToolsOpened) throw new Error("DevTools is open"); + debuggee.attached = true; + }, + detach: vi.fn(() => { + debuggee.attached = false; + }), }); let destroyed = false; + let devToolsOpened = false; const webContents = Object.assign(new NodeEvents.EventEmitter(), { + debugger: debuggee, getURL: () => "http://localhost/", getTitle: () => "Page", getUserAgent: () => "Electron", isLoadingMainFrame: () => options.loading === true, isDestroyed: () => destroyed, + isDevToolsOpened: () => devToolsOpened, + close: vi.fn(), }); return { tab: { @@ -85,11 +99,22 @@ const makeDebuggee = ( debugger: debuggee as unknown as Electron.Debugger, }, emit: (method: string, params: unknown) => emitter.emit("message", {}, method, params, ""), - navigate: () => webContents.emit("did-navigate"), - domReady: () => webContents.emit("dom-ready"), + debuggee, + openDevTools: () => { + devToolsOpened = true; + debuggee.attached = false; + debuggee.emit("detach", {}, "replaced_with_devtools"); + }, + closeDevTools: () => { + devToolsOpened = false; + webContents.emit("devtools-closed"); + }, destroy: () => { destroyed = true; + webContents.emit("destroyed"); }, + navigate: () => webContents.emit("did-navigate"), + domReady: () => webContents.emit("dom-ready"), failLoad: (code: number) => webContents.emit("did-fail-load", {}, code, "Failed", "http://localhost/", true), /** Ids of the commands the debugger received, in order. */ @@ -123,9 +148,11 @@ const watchReplies = (host: DesktopBrowserHost.DesktopBrowserHost["Service"]) => const replies = yield* Queue.unbounded(); yield* host.events.pipe( Stream.map((line) => decodeEvent(new TextDecoder().decode(line))), - Stream.runForEach((event) => - event.type === "cdp" ? Queue.offer(replies, decodeCdpReply(event.message).id) : Effect.void, - ), + Stream.runForEach((event) => { + if (event.type !== "cdp") return Effect.void; + const { id } = decodeCdpReply(event.message); + return id === undefined ? Effect.void : Queue.offer(replies, id); + }), Effect.forkScoped, ); yield* Effect.yieldNow; @@ -160,17 +187,19 @@ describe("DesktopBrowserHost", () => { const host = yield* DesktopBrowserHost.make; const loading = makeDebuggee({ loading: true }); const other = { ...key, tabId: "tab-2" }; - const reader = yield* takeEvents(host, 3).pipe(Effect.forkScoped); + const reader = yield* takeEvents(host, 4).pipe(Effect.forkScoped); yield* Effect.yieldNow; host.attach(key, loading.tab); host.setShown(key, true); + host.openedPopup(loading.tab.webContents, makeDebuggee().tab.webContents); host.attach(other, makeDebuggee().tab); loading.domReady(); expect(yield* Fiber.join(reader)).toEqual([ { type: "attached", ...other }, { type: "attached", ...key }, + { type: "popupWindows", ...key, open: true }, { type: "shown", ...key, shown: true }, ]); }), @@ -209,7 +238,9 @@ describe("DesktopBrowserHost", () => { const reader = yield* takeEvents(host, 1).pipe(Effect.forkScoped); yield* Effect.yieldNow; - host.attach(closed, makeDebuggee({ loading: true }).tab); + const closedGuest = makeDebuggee({ loading: true }); + host.attach(closed, closedGuest.tab); + host.openedPopup(closedGuest.tab.webContents, makeDebuggee().tab.webContents); host.detach(closed); host.attach(destroyed, destroyedGuest.tab); destroyedGuest.destroy(); @@ -223,19 +254,25 @@ describe("DesktopBrowserHost", () => { it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; - host.attach(key, makeDebuggee().tab); + const page = makeDebuggee(); + const popup = makeDebuggee(); + popup.openDevTools(); + host.openedPopup(page.tab.webContents, popup.tab.webContents); + host.attach(key, page.tab); host.setShown(key, true); - const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped); + const reader = yield* takeEvents(host, 7).pipe(Effect.forkScoped); yield* Effect.yieldNow; host.detach(key); - host.attach(key, makeDebuggee().tab); + host.attach(key, page.tab); expect(yield* Fiber.join(reader)).toEqual([ { type: "attached", ...key }, + { type: "popupWindows", ...key, open: true }, { type: "shown", ...key, shown: true }, { type: "detached", ...key }, { type: "attached", ...key }, + { type: "popupWindows", ...key, open: true }, { type: "shown", ...key, shown: true }, ]); }), @@ -245,13 +282,16 @@ describe("DesktopBrowserHost", () => { Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; const other = { ...key, tabId: "tab-2" }; + const page = makeDebuggee(); + host.openedPopup(page.tab.webContents, makeDebuggee().tab.webContents); host.setShown(key, false); - host.attach(key, makeDebuggee().tab); + host.attach(key, page.tab); host.setShown(other, true); host.attach(other, makeDebuggee().tab); - expect(yield* takeEvents(host, 3)).toEqual([ + expect(yield* takeEvents(host, 4)).toEqual([ { type: "attached", ...key }, + { type: "popupWindows", ...key, open: true }, { type: "shown", ...key, shown: false }, { type: "attached", ...other }, ]); @@ -774,6 +814,7 @@ describe("DesktopBrowserHost", () => { yield* settle; debuggee.release(); expect(yield* Queue.take(replies)).toBe(1); + debuggee.emit("Target.attachedToTarget", { sessionId: "child-frame" }); yield* raw({ id: 2, method: "Page.startScreencast", sessionId: "child-frame" }); expect(yield* Queue.take(replies)).toBe(2); yield* TestClock.adjust("1 minute"); @@ -799,11 +840,13 @@ describe("DesktopBrowserHost", () => { }); expect(yield* Queue.take(replies)).toBe(2); yield* raw({ id: 3, method: "Page.startScreencast", sessionId: "t3-preview-page-1" }); + expect(yield* Queue.take(replies)).toBe(3); yield* settle; debuggee.release(); yield* settle; yield* TestClock.adjust("1 minute"); - expect(debuggee.focusEmulation).toEqual([true, false]); + expect(debuggee.focusEmulation).toEqual([]); + expect(debuggee.sent).toEqual([]); }).pipe(Effect.scoped), ); @@ -1083,6 +1126,7 @@ describe("DesktopBrowserHost", () => { id: 2, method: "Input.dispatchMouseEvent", params: { type: "mousePressed", x: 1, y: 1 }, + sessionId: "t3-preview-page", }), }), ); @@ -1090,6 +1134,255 @@ describe("DesktopBrowserHost", () => { }), ); + it.effect("drives an agent tab's popup windows through its relay, each on its own", () => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const page = makeDebuggee(); + const early = makeDebuggee({ targetId: "EARLY" }); + const late = makeDebuggee({ targetId: "LATE" }); + const human = makeDebuggee(); + const humanPopup = makeDebuggee({ targetId: "HUMAN-POPUP" }); + const humanKey = { threadId: "thread-1", tabId: "tab-human" }; + const events = yield* Queue.unbounded(); + yield* host.events.pipe( + Stream.runForEach((line) => + Queue.offer(events, decodeEvent(new TextDecoder().decode(line))), + ), + Effect.forkScoped, + ); + const cdp = (tabKey: typeof key, message: Record) => + host.handleCommandLine( + encodeJson({ type: "cdp", ...tabKey, message: encodeJson(message) }), + ); + const nextCdp = Effect.gen(function* () { + let event = yield* Queue.take(events); + while (event.type !== "cdp") event = yield* Queue.take(events); + return JSON.parse(event.message) as Record; + }); + host.attach(key, page.tab); + host.attach(humanKey, human.tab); + yield* Queue.takeN(events, 2); + host.openedPopup(page.tab.webContents, early.tab.webContents); + host.openedPopup(human.tab.webContents, humanPopup.tab.webContents); + + yield* host.handleCommandLine(encodeJson({ type: "popups", ...key })); + yield* cdp(key, { id: 1, method: "Target.setAutoAttach" }); + yield* cdp(humanKey, { id: 1, method: "Target.setAutoAttach" }); + const announced: Array> = []; + while (announced.length < 3) { + const message = yield* nextCdp; + if (message["method"] === "Target.attachedToTarget") announced.push(message); + } + host.openedPopup(page.tab.webContents, late.tab.webContents); + let lateAnnounced = yield* nextCdp; + while (lateAnnounced["method"] !== "Target.attachedToTarget") lateAnnounced = yield* nextCdp; + announced.push(lateAnnounced); + const popups = announced + .map((message) => message["params"] as { sessionId: string; targetInfo: object }) + .filter((params) => params.sessionId !== "t3-preview-page"); + expect(popups.map((params) => params.targetInfo)).toEqual([ + expect.objectContaining({ targetId: "EARLY", openerId: "GUEST" }), + expect.objectContaining({ targetId: "LATE", openerId: "GUEST" }), + ]); + // A person's tab keeps its popups to itself. + expect(humanPopup.debuggee.attached).toBe(false); + expect(early.debuggee.attached).toBe(true); + + const [earlySession, lateSession] = popups.map((params) => params.sessionId); + yield* cdp(key, { + id: 2, + method: "Page.captureScreenshot", + params: { id: 2 }, + sessionId: earlySession, + }); + yield* cdp(key, { + id: 3, + method: "Page.captureScreenshot", + params: { id: 3 }, + sessionId: lateSession, + }); + yield* settle; + // The early popup's pending screenshot neither blocks the late one's nor focuses the page. + expect([early.sent, late.sent, page.sent]).toEqual([[2], [3], []]); + expect([early.focusEmulation, late.focusEmulation, page.focusEmulation]).toEqual([ + [true], + [true], + [], + ]); + + yield* cdp(key, { + id: 4, + method: "Browser.setDownloadBehavior", + params: { behavior: "allowAndName", downloadPath: "/srv/downloads" }, + }); + yield* settle; + for (const debuggee of [page, early, late]) debuggee.release(); + let reply = yield* nextCdp; + while (reply["id"] !== 4) reply = yield* nextCdp; + const paths: Array = []; + const item = { + setSavePath: (path: string) => void paths.push(path), + } as unknown as Electron.DownloadItem; + late.emit("Page.downloadWillBegin", { guid: "guid-late" }); + expect(host.placeDownload(page.tab.webContents, item)).toBe(false); + expect(host.placeDownload(late.tab.webContents, item)).toBe(true); + expect(paths).toEqual(["/srv/downloads/guid-late"]); + + for (const id of [5, 6, 7]) { + yield* cdp(key, { + id, + method: id === 7 ? "Page.startScreencast" : "Page.captureScreenshot", + params: { id }, + sessionId: earlySession, + }); + } + yield* settle; + yield* cdp(key, { + id: 8, + method: "Target.detachFromTarget", + params: { sessionId: earlySession }, + }); + let detached = yield* nextCdp; + while (detached["id"] !== 8) detached = yield* nextCdp; + yield* TestClock.adjust("1 minute"); + expect(early.focusEmulation).toEqual([true, false]); + early.release(); + let queued = yield* nextCdp; + while (queued["id"] !== 6) queued = yield* nextCdp; + expect(queued).toHaveProperty("error"); + expect(early.sent.filter((id) => id === 5 || id === 6)).toEqual([5]); + expect(early.tab.webContents.close).not.toHaveBeenCalled(); + + late.destroy(); + let closed = yield* nextCdp; + while (closed["method"] !== "Target.detachedFromTarget") closed = yield* nextCdp; + expect(closed["params"]).toEqual({ sessionId: lateSession, targetId: "LATE" }); + + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + expect(early.debuggee.detach).toHaveBeenCalledOnce(); + expect(late.debuggee.detach).not.toHaveBeenCalled(); + }).pipe(Effect.scoped), + ); + + it.effect.each(["before", "after"] as const)( + "recovers the same popup when DevTools opened %s relay attachment closes", + (when) => + Effect.gen(function* () { + const host = yield* DesktopBrowserHost.make; + const page = makeDebuggee(); + const popup = makeDebuggee({ targetId: "POPUP" }); + const sibling = makeDebuggee({ targetId: "SIBLING" }); + const attach = vi.spyOn(popup.debuggee, "attach"); + const nativeStates: Array = []; + const events = yield* Queue.unbounded(); + yield* host.events.pipe( + Stream.runForEach((line) => { + const event = decodeEvent(new TextDecoder().decode(line)); + if (event.type === "popupWindows") nativeStates.push(event.open); + return Queue.offer(events, event); + }), + Effect.forkScoped, + ); + const cdp = (message: Record) => + host.handleCommandLine(encodeJson({ type: "cdp", ...key, message: encodeJson(message) })); + const nextCdp = Effect.gen(function* () { + let event = yield* Queue.take(events); + while (event.type !== "cdp") event = yield* Queue.take(events); + return JSON.parse(event.message) as Record; + }); + const reply = (id: number) => + Effect.gen(function* () { + let message = yield* nextCdp; + while (message["id"] !== id) message = yield* nextCdp; + return message; + }); + host.attach(key, page.tab); + expect(yield* Queue.take(events)).toEqual({ type: "attached", ...key }); + if (when === "before") popup.openDevTools(); + host.openedPopup(page.tab.webContents, popup.tab.webContents); + host.openedPopup(page.tab.webContents, sibling.tab.webContents); + yield* host.handleCommandLine(encodeJson({ type: "popups", ...key })); + yield* cdp({ id: 1, method: "Target.setAutoAttach" }); + const sessions = new Map(); + while (sessions.size < (when === "before" ? 2 : 3)) { + const message = yield* nextCdp; + if (message["method"] !== "Target.attachedToTarget") continue; + const params = message["params"] as { + sessionId: string; + targetInfo: { targetId: string }; + }; + sessions.set(params.targetInfo.targetId, params.sessionId); + } + if (when === "after") { + popup.openDevTools(); + const message = yield* nextCdp; + expect(message).toMatchObject({ + method: "Target.detachedFromTarget", + params: { sessionId: sessions.get("POPUP"), targetId: "POPUP" }, + }); + } + expect(popup.debuggee.attached).toBe(false); + expect(nativeStates).toEqual([true]); + attach.mockClear(); + popup.closeDevTools(); + expect(popup.debuggee.attached).toBe(true); + let restored = yield* nextCdp; + while (restored["method"] !== "Target.attachedToTarget") restored = yield* nextCdp; + const restoredParams = restored["params"] as { sessionId: string }; + expect(restored).toMatchObject({ + params: { targetInfo: { targetId: "POPUP", openerId: "GUEST" } }, + }); + expect(restoredParams.sessionId).not.toBe(sessions.get("POPUP")); + popup.closeDevTools(); + expect(attach).toHaveBeenCalledOnce(); + if (when === "after") { + yield* cdp({ id: 2, method: "Runtime.evaluate", sessionId: sessions.get("POPUP") }); + expect(yield* reply(2)).toHaveProperty("error"); + } + for (const [index, sessionId] of [ + restoredParams.sessionId, + sessions.get("SIBLING"), + sessions.get("GUEST"), + ].entries()) { + yield* cdp({ + id: index + 3, + method: "Runtime.evaluate", + params: { id: index + 3 }, + sessionId, + }); + } + yield* settle; + expect([popup.sent, sibling.sent, page.sent]).toEqual([[3], [4], [5]]); + popup.release(); + sibling.release(); + page.release(); + yield* reply(5); + yield* cdp({ id: 6, method: "Target.getTargets" }); + expect(yield* reply(6)).toMatchObject({ + result: { + targetInfos: ["GUEST", "SIBLING", "POPUP"].map((targetId) => + expect.objectContaining({ targetId, attached: true }), + ), + }, + }); + const siblingAttach = vi.spyOn(sibling.debuggee, "attach"); + sibling.openDevTools(); + sibling.destroy(); + sibling.closeDevTools(); + expect(siblingAttach).not.toHaveBeenCalled(); + yield* host.handleCommandLine(encodeJson({ type: "release", ...key })); + popup.closeDevTools(); + expect(attach).toHaveBeenCalledOnce(); + popup.destroy(); + popup.closeDevTools(); + let closed = yield* Queue.take(events); + while (closed.type !== "popupWindows" || closed.open) closed = yield* Queue.take(events); + expect(nativeStates).toEqual([true, false]); + expect(attach).toHaveBeenCalledOnce(); + expect(popup.tab.webContents.close).not.toHaveBeenCalled(); + }).pipe(Effect.scoped), + ); + it.effect("detaches only the tab whose CDP event is too large to relay", () => Effect.gen(function* () { const host = yield* DesktopBrowserHost.make; diff --git a/apps/desktop/src/preview/DesktopBrowserHost.ts b/apps/desktop/src/preview/DesktopBrowserHost.ts index 2168532902e15..95acb2e9b7d68 100644 --- a/apps/desktop/src/preview/DesktopBrowserHost.ts +++ b/apps/desktop/src/preview/DesktopBrowserHost.ts @@ -15,15 +15,17 @@ import * as PubSub from "effect/PubSub"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; -import { createCdpRelayConnection, type CdpRelayConnection } from "./CdpRelay.ts"; +import { + createCdpRelayConnection, + type CdpRelayConnection, + type CdpRelayPopup, + type CdpRelayTarget, +} from "./CdpRelay.ts"; import { sendPageCommand } from "./GuestInput.ts"; const encodeEvent = Schema.encodeSync(Schema.fromJsonString(DesktopBrowserEvent)); const decodeCommand = Schema.decodeUnknownOption(Schema.fromJsonString(DesktopBrowserCommand)); const lineEncoder = new TextEncoder(); -const AGENT_INPUT_COMMAND = - /"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/; -/** The same commands as above, by method, for the relay's sends. */ const AGENT_INPUT_METHOD = /^(?:Input\.|Page\.navigate$|Runtime\.(?:evaluate|callFunctionOn)$)/; const AGENT_DOWNLOAD_WINDOW_MS = 5_000; const FOCUS_IDLE = "5 seconds"; @@ -43,10 +45,10 @@ export interface DesktopBrowserTabDebugger { const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000${tabId}`; -interface AttachedTab { - readonly key: DesktopBrowserTabKey; +/** A page the relay drives: the tab's own, or a popup it opened. */ +interface AttachedTarget { + readonly tab: AttachedTab; readonly debuggee: DesktopBrowserTabDebugger; - relay: CdpRelayConnection | null; screenshotTurn: object | null; /** Each holds focus and painting while `held` and whoever queued it still waits on it. */ readonly waitingScreenshots: Array<{ @@ -65,8 +67,6 @@ interface AttachedTab { object, { readonly owner: string | undefined; readonly paints: boolean } >; - /** Whether the client keeps the guest painting, as published on `painting`. */ - painting: boolean; /** * Driven by the agent since its thread's last turn ended or the guest last throttled. A * turn's start can reach us after its first command, so this holds focus once it does. @@ -82,10 +82,33 @@ interface AttachedTab { sessionId: string, ) => void; readonly onNewDocument: () => void; +} + +/** Sessions of one relay connection, which its page and popups share. */ +interface RelayOwners { + readonly detached: Set; + /** Viewers and recordings stream; the agent's own sessions never do. */ + readonly streaming: Set; +} + +interface AttachedTab extends AttachedTarget { + readonly key: DesktopBrowserTabKey; + relay: CdpRelayConnection | null; + owners: RelayOwners; + /** Whether the client keeps the guest painting, as published on `painting`. */ + painting: boolean; + /** Set while the server drives the tab for an agent, whose popups become its tabs too. */ + showsPopups: boolean; + readonly popups: Map; /** Stops waiting for the first page; set until the tab is announced. */ stopWaiting: (() => void) | null; } +interface AttachedPopup extends AttachedTarget { + relayed: CdpRelayPopup | null; + readonly onDetach: () => void; +} + export class DesktopBrowserHost extends Context.Service< DesktopBrowserHost, { @@ -93,6 +116,8 @@ export class DesktopBrowserHost extends Context.Service< readonly handleCommandLine: (line: string) => Effect.Effect; readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void; readonly detach: (key: DesktopBrowserTabKey) => void; + /** A guest opened a popup window; a tab the server drives for an agent shows it as a tab too. */ + readonly openedPopup: (opener: Electron.WebContents, popup: Electron.WebContents) => void; readonly setShown: (key: DesktopBrowserTabKey, shown: boolean | undefined) => void; readonly humanStartedDownload: (source: Electron.WebContents) => boolean; readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean; @@ -124,6 +149,8 @@ export const make = Effect.gen(function* () { }>(); const runFork = Effect.runForkWith(yield* Effect.context()); const tabs = new Map(); + /** Every guest's open popups, so a tab the server starts driving later still shows them. */ + const openedPopups = new Map>(); /** Threads whose agent turn runs, by the server's account. */ const agentTurns = new Set(); // Kept apart from attachment: the renderer can show a webview before its debugger attaches, @@ -131,11 +158,15 @@ export const make = Effect.gen(function* () { const shownTabs = new Map(); const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event)); - const nextScreenshot = (tab: AttachedTab) => { - tab.screenshotTurn = null; - tab.waitingScreenshots.shift()?.start(); - releaseFocusWhenIdle(tab); - syncPainting(tab); + const isAttached = (target: AttachedTarget) => + tabs.get(keyOf(target.tab.key)) === target.tab && + (target === target.tab || target.tab.popups.get(target.debuggee.webContents) === target); + + const nextScreenshot = (target: AttachedTarget) => { + target.screenshotTurn = null; + target.waitingScreenshots.shift()?.start(); + releaseFocusWhenIdle(target); + syncPainting(target); }; /** @@ -145,38 +176,38 @@ export const make = Effect.gen(function* () { * gets one at a time. One whose relay or session has since gone is dropped unsent. */ const takeScreenshot = ( - tab: AttachedTab, + target: AttachedTarget, relay: CdpRelayConnection, ownerLive: () => boolean, /** Sends only while `current` holds; holds focus and painting only while `held`. */ capture: (current: () => boolean, held: boolean) => Promise, ) => new Promise((resolve, reject) => { - const live = () => tab.relay === relay && ownerLive(); + const live = () => target.tab.relay === relay && isAttached(target) && ownerLive(); const waiting = { start: () => { if (!live()) { reject(new Error("The preview connection that asked for this screenshot closed.")); - nextScreenshot(tab); + nextScreenshot(target); return; } const turn = {}; - tab.screenshotTurn = turn; - capture(() => tab.screenshotTurn === turn && live(), waiting.held) + target.screenshotTurn = turn; + capture(() => target.screenshotTurn === turn && live(), waiting.held) .finally(() => { - if (tab.screenshotTurn === turn) nextScreenshot(tab); + if (target.screenshotTurn === turn) nextScreenshot(target); }) .then(resolve, reject); }, live, held: true, }; - if (tab.screenshotTurn === null) waiting.start(); + if (target.screenshotTurn === null) waiting.start(); else { // The one ahead may be abandoned mid-capture; it needs frames to finish and free its turn. - tab.waitingScreenshots.push(waiting); - setFocusEmulation(tab, true); - syncPainting(tab); + target.waitingScreenshots.push(waiting); + setFocusEmulation(target, true); + syncPainting(target); } }); @@ -185,44 +216,44 @@ export const make = Effect.gen(function* () { * never throttle. Without it, a hidden guest paints no frames and focus() fires no focus * events, so the guest is emulated focused while the server drives it or streams it. */ - const setFocusEmulation = (tab: AttachedTab, enabled: boolean) => { - if (tab.focusEmulated === enabled) return; - tab.focusEmulated = enabled; + const setFocusEmulation = (target: AttachedTarget, enabled: boolean) => { + if (target.focusEmulated === enabled) return; + target.focusEmulated = enabled; // A guest that refuses it, or a debugger already gone, behaves as unfocused. - const change: Promise = tab.debuggee.debugger + const change: Promise = target.debuggee.debugger .sendCommand("Emulation.setFocusEmulationEnabled", { enabled }) .then( () => undefined, () => undefined, ) .finally(() => { - if (tab.focusChange === change) tab.focusChange = null; + if (target.focusChange === change) target.focusChange = null; }); - tab.focusChange = change; + target.focusChange = change; }; - const waitingHolds = (tab: AttachedTab) => - tab.waitingScreenshots.some((waiting) => waiting.held && waiting.live()); + const waitingHolds = (target: AttachedTarget) => + target.waitingScreenshots.some((waiting) => waiting.held && waiting.live()); - const releaseFocusWhenIdle = (tab: AttachedTab) => { - if (tab.focusReleasing) return; - tab.focusReleasing = true; + const releaseFocusWhenIdle = (target: AttachedTarget) => { + if (target.focusReleasing) return; + target.focusReleasing = true; runFork( Effect.gen(function* () { let uses: number; do { - uses = tab.focusUses; + uses = target.focusUses; yield* Effect.sleep(FOCUS_IDLE); - } while (uses !== tab.focusUses); - tab.focusReleasing = false; + } while (uses !== target.focusUses); + target.focusReleasing = false; // The last pending command to settle starts another idle wait. - if (tab.focusPending.size > 0 || tab.screencasts.size > 0 || waitingHolds(tab)) { + if (target.focusPending.size > 0 || target.screencasts.size > 0 || waitingHolds(target)) { return; } // The agent pauses to think between steps; its tab stays focused until the turn ends. - if (tab.agentUsed && agentTurns.has(tab.key.threadId)) return; - tab.agentUsed = false; - if (tabs.get(keyOf(tab.key)) === tab) setFocusEmulation(tab, false); + if (target.agentUsed && agentTurns.has(target.tab.key.threadId)) return; + target.agentUsed = false; + if (isAttached(target)) setFocusEmulation(target, false); }), ); }; @@ -231,8 +262,11 @@ export const make = Effect.gen(function* () { * Electron composites no offscreen guest, so its screenshot or screencast waits for a frame * that never comes. The client paints the guest behind the app while one is pending; the * capture then finishes on its own, and the guest goes back offscreen once none is. + * A popup is a window of its own, which the client does not paint. */ - const syncPainting = (tab: AttachedTab) => { + const syncPainting = (target: AttachedTarget) => { + const tab = target.tab; + if (target !== tab) return; const active = tabs.get(keyOf(tab.key)) === tab && (tab.screencasts.size > 0 || @@ -245,22 +279,22 @@ export const make = Effect.gen(function* () { /** A command may wait on the page's next frame, so the guest stays focused until it settles. */ const holdFocus = ( - tab: AttachedTab, + target: AttachedTarget, method: string, owner: string | undefined, work: () => Promise, ) => { const pending = {}; - tab.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); - tab.focusUses += 1; - setFocusEmulation(tab, true); - releaseFocusWhenIdle(tab); - syncPainting(tab); + target.focusPending.set(pending, { owner, paints: method === "Page.captureScreenshot" }); + target.focusUses += 1; + setFocusEmulation(target, true); + releaseFocusWhenIdle(target); + syncPainting(target); return work().finally(() => { - if (!tab.focusPending.delete(pending)) return; - tab.focusUses += 1; - releaseFocusWhenIdle(tab); - syncPainting(tab); + if (!target.focusPending.delete(pending)) return; + target.focusUses += 1; + releaseFocusWhenIdle(target); + syncPainting(target); }); }; @@ -269,27 +303,29 @@ export const make = Effect.gen(function* () { * also end every hold: a released relay has no one waiting, and a new document answers * nothing pending from the old one. A released relay's screencast has no viewer to stop it. */ - const dropFocusHolds = (tab: AttachedTab) => { - tab.focusPending.clear(); - releaseFocusWhenIdle(tab); - syncPainting(tab); + const dropFocusHolds = (target: AttachedTarget) => { + target.focusPending.clear(); + releaseFocusWhenIdle(target); + syncPainting(target); }; const dropRelay = (tab: AttachedTab) => { tab.relay = null; + tab.showsPopups = false; + for (const popup of tab.popups.values()) hidePopup(popup); tab.screencasts.clear(); tab.agentUsed = false; dropFocusHolds(tab); }; /** A detached session, such as a closed viewer, waits on nothing it sent. */ - const dropSessionHolds = (tab: AttachedTab, owner: string) => { - tab.screencasts.delete(owner); - for (const [pending, hold] of tab.focusPending) { - if (hold.owner === owner) tab.focusPending.delete(pending); + const dropSessionHolds = (target: AttachedTarget, owner: string) => { + target.screencasts.delete(owner); + for (const [pending, hold] of target.focusPending) { + if (hold.owner === owner) target.focusPending.delete(pending); } - releaseFocusWhenIdle(tab); - syncPainting(tab); + releaseFocusWhenIdle(target); + syncPainting(target); }; /** @@ -301,80 +337,96 @@ export const make = Effect.gen(function* () { agentTurns.delete(threadId); for (const tab of tabs.values()) { if (tab.key.threadId !== threadId) continue; - tab.agentUsed = false; - // Still sent in turn, so the queue keeps moving, but no longer kept painting. - for (const waiting of tab.waitingScreenshots) waiting.held = false; - dropFocusHolds(tab); + for (const target of [tab, ...tab.popups.values()]) { + target.agentUsed = false; + // Still sent in turn, so the queue keeps moving, but no longer kept painting. + for (const waiting of target.waitingScreenshots) waiting.held = false; + dropFocusHolds(target); + } } }; - const relayFor = (tab: AttachedTab) => { - if (tab.relay) return tab.relay; - const { webContents, debugger: debuggee } = tab.debuggee; + /** One relay drives the tab's page and its popups; each keeps its own focus and screenshots. */ + const relayTarget = ( + target: AttachedTarget, + relay: () => CdpRelayConnection, + owners: RelayOwners, + ): CdpRelayTarget => { + const { webContents, debugger: debuggee } = target.debuggee; + const live = () => target.tab.relay === relay() && isAttached(target); const send = (method: string, params: Record, sessionId?: string) => sendPageCommand(webContents, debuggee, method, params, sessionId); - const detachedOwners = new Set(); - /** Viewers and recordings stream; the agent's own sessions never do. */ - const streamingOwners = new Set(); - const relay: CdpRelayConnection = createCdpRelayConnection( - { - send: (method, params, sessionId, owner) => { - // A child session's screencast is not the page's, such as a late one on a detached session. - if (sessionId === undefined && method === "Page.startScreencast") { - tab.screencasts.add(owner); - if (owner !== undefined) streamingOwners.add(owner); - } else if (sessionId === undefined && method === "Page.stopScreencast") { - tab.screencasts.delete(owner); - } + return { + send: (method, params, sessionId, owner) => { + // A child session's screencast is not the page's, such as a late one on a detached session. + if (sessionId === undefined && method === "Page.startScreencast") { + target.screencasts.add(owner); + if (owner !== undefined) owners.streaming.add(owner); + } else if (sessionId === undefined && method === "Page.stopScreencast") { + target.screencasts.delete(owner); + } + if (AGENT_INPUT_METHOD.test(method)) { + target.agentInputAt = performance.now(); // A person's input through a viewer, or Playwright's own housekeeping, is not the agent's. - if ( - AGENT_INPUT_METHOD.test(method) && - (owner === undefined || !streamingOwners.has(owner)) - ) { - tab.agentUsed = true; - } - // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. - const forward = (current?: () => boolean) => { - const change = tab.focusChange; - const run = () => - tab.relay === relay && (current === undefined || current()) - ? send(method, params, sessionId) - : Promise.reject(new Error("The page this screenshot was for has moved on.")); - return change === null ? run() : change.then(run); - }; - return method === "Page.captureScreenshot" - ? takeScreenshot( - tab, - relay, - () => owner === undefined || !detachedOwners.has(owner), - (current, held) => - held ? holdFocus(tab, method, owner, () => forward(current)) : forward(current), - ) - : method === "Page.startScreencast" - ? holdFocus(tab, method, owner, forward).catch((cause: unknown) => { - // A viewer can outlive a refused stream; nothing is streaming for it. - if (sessionId === undefined && tab.relay === relay) tab.screencasts.delete(owner); - releaseFocusWhenIdle(tab); - syncPainting(tab); - throw cause; - }) - : holdFocus(tab, method, owner, forward); - }, - sessionDetached: (owner) => { - detachedOwners.add(owner); - dropSessionHolds(tab, owner); - }, - targetId: () => - debuggee - .sendCommand("Target.getTargetInfo") - .then((result: { targetInfo: { targetId: string } }) => result.targetInfo.targetId), - url: () => webContents.getURL(), - title: () => webContents.getTitle(), - userAgent: () => webContents.getUserAgent(), - setDownloadDirectory: (directory) => { - tab.downloadDirectory = directory; - }, + if (owner === undefined || !owners.streaming.has(owner)) target.agentUsed = true; + } + // Waits for focus emulation; insertText and other non-CDP input would not queue behind it. + const forward = (current?: () => boolean) => { + const change = target.focusChange; + const run = () => + live() && (current === undefined || current()) + ? send(method, params, sessionId) + : Promise.reject(new Error("The page this screenshot was for has moved on.")); + return change === null ? run() : change.then(run); + }; + return method === "Page.captureScreenshot" + ? takeScreenshot( + target, + relay(), + () => owner === undefined || !owners.detached.has(owner), + (current, held) => + held ? holdFocus(target, method, owner, () => forward(current)) : forward(current), + ) + : method === "Page.startScreencast" + ? holdFocus(target, method, owner, forward).catch((cause: unknown) => { + // A viewer can outlive a refused stream; nothing is streaming for it. + if (sessionId === undefined && live()) target.screencasts.delete(owner); + releaseFocusWhenIdle(target); + syncPainting(target); + throw cause; + }) + : holdFocus(target, method, owner, forward); + }, + sessionDetached: (owner) => { + owners.detached.add(owner); + dropSessionHolds(target, owner); }, + targetId: () => + debuggee + .sendCommand("Target.getTargetInfo") + .then((result: { targetInfo: { targetId: string } }) => result.targetInfo.targetId), + url: () => webContents.getURL(), + title: () => webContents.getTitle(), + userAgent: () => webContents.getUserAgent(), + setDownloadDirectory: (directory) => { + target.downloadDirectory = directory; + }, + ...(target === target.tab + ? {} + : { + close: () => { + if (!webContents.isDestroyed()) webContents.close(); + }, + }), + }; + }; + + const relayFor = (tab: AttachedTab) => { + if (tab.relay) return tab.relay; + const { debugger: debuggee } = tab.debuggee; + tab.owners = { detached: new Set(), streaming: new Set() }; + const relay: CdpRelayConnection = createCdpRelayConnection( + relayTarget(tab, () => relay, tab.owners), (message) => { if (tab.relay === relay && tabs.get(keyOf(tab.key)) === tab) { emit({ type: "cdp", ...tab.key, message }); @@ -400,17 +452,144 @@ export const make = Effect.gen(function* () { }, ); tab.relay = relay; + showPopups(tab); return relay; }; + const freshTarget = (debuggee: DesktopBrowserTabDebugger) => ({ + debuggee, + screenshotTurn: null, + waitingScreenshots: [], + downloadDirectory: null, + pendingDownloadGuid: null, + agentInputAt: Number.NEGATIVE_INFINITY, + focusEmulated: false, + focusChange: null, + focusUses: 0, + focusPending: new Map(), + agentUsed: false, + focusReleasing: false, + screencasts: new Set(), + }); + + const listen = >( + target: T, + relayed: () => { readonly event: CdpRelayConnection["event"] } | null, + ): T & Pick => + Object.assign(target, { + onMessage: (_event: Electron.Event, method: string, params: unknown, sessionId: string) => { + if (method === "Page.downloadWillBegin") { + const guid = (params as { guid?: unknown } | undefined)?.guid; + target.pendingDownloadGuid = typeof guid === "string" ? guid : null; + } + relayed()?.event(method, params, sessionId); + }, + // A screenshot pending when the page loads a new document, a reload included, can + // stay unanswered. A crashed page lands here once it reloads, well after its renderer died. + onNewDocument: () => { + const listening = target as T & AttachedTarget; + // Before the next screenshot starts, so its own hold survives. + dropFocusHolds(listening); + if (listening.screenshotTurn !== null) nextScreenshot(listening); + }, + }); + + const showPopup = (tab: AttachedTab, webContents: Electron.WebContents) => { + const relay = tab.relay; + if (!relay || !tab.showsPopups || tab.popups.has(webContents) || webContents.isDestroyed()) + return; + // Kept for the popup's lifetime; `webContents.debugger` throws once it is destroyed. + const debuggee = webContents.debugger; + if (webContents.isDevToolsOpened() || debuggee.isAttached()) return; + try { + debuggee.attach("1.3"); + } catch { + return; + } + const popup: AttachedPopup = listen( + { + ...freshTarget({ webContents, debugger: debuggee }), + tab, + relayed: null, + onDetach: () => hidePopup(popup, true), + }, + () => popup.relayed, + ); + tab.popups.set(webContents, popup); + debuggee.on("message", popup.onMessage); + debuggee.on("detach", popup.onDetach); + webContents.on("did-navigate", popup.onNewDocument); + popup.relayed = relay.addPopup(relayTarget(popup, () => relay, tab.owners)); + }; + + /** Detaching the debugger also ends the popup's streams, emulation and paused children. */ + const hidePopup = (popup: AttachedPopup, debuggerGone = false) => { + const { tab, debuggee } = popup; + if (tab.popups.get(debuggee.webContents) !== popup) return; + tab.popups.delete(debuggee.webContents); + debuggee.debugger.off("message", popup.onMessage); + debuggee.debugger.off("detach", popup.onDetach); + debuggee.webContents.off("did-navigate", popup.onNewDocument); + popup.relayed?.remove(); + popup.screencasts.clear(); + dropFocusHolds(popup); + if (!debuggerGone && !debuggee.webContents.isDestroyed() && debuggee.debugger.isAttached()) { + debuggee.debugger.detach(); + } + }; + + const showPopups = (tab: AttachedTab) => { + for (const popup of openedPopups.get(tab.debuggee.webContents) ?? []) showPopup(tab, popup); + }; + + const openedPopup = (opener: Electron.WebContents, popup: Electron.WebContents) => { + if (popup.isDestroyed()) return; + const popups = openedPopups.get(opener) ?? new Set(); + if (popups.has(popup)) return; + openedPopups.set(opener, popups); + popups.add(popup); + const show = () => { + for (const tab of tabs.values()) + if (tab.debuggee.webContents === opener) showPopup(tab, popup); + }; + const report = (open: boolean) => { + for (const tab of tabs.values()) { + // A tab still waiting for its first page reports them when it is announced. + if (tab.debuggee.webContents === opener && tab.stopWaiting === null) { + emit({ type: "popupWindows", ...tab.key, open }); + } + } + }; + popup.on("devtools-closed", show); + if (popups.size === 1) report(true); + popup.once("destroyed", () => { + popup.off("devtools-closed", show); + popups.delete(popup); + if (popups.size === 0 && openedPopups.get(opener) === popups) openedPopups.delete(opener); + if (popups.size === 0) report(false); + for (const tab of tabs.values()) { + const shown = tab.popups.get(popup); + if (shown) hidePopup(shown, true); + } + }); + show(); + }; + + const targetOf = (source: Electron.WebContents) => { + for (const tab of tabs.values()) { + if (tab.debuggee.webContents === source) return tab; + const popup = tab.popups.get(source); + if (popup) return popup; + } + return undefined; + }; + // CDP supplies the path before this runs; otherwise Electron opens a Save dialog. const placeDownload = (source: Electron.WebContents, item: Electron.DownloadItem) => { - const tab = [...tabs.values()].find( - (candidate) => candidate.debuggee.webContents === source && candidate.downloadDirectory, - ); - if (!tab?.downloadDirectory || !tab.pendingDownloadGuid) return false; - item.setSavePath(NodePath.join(tab.downloadDirectory, tab.pendingDownloadGuid)); - tab.pendingDownloadGuid = null; + const target = targetOf(source); + if (!target?.downloadDirectory || !target.pendingDownloadGuid) return false; + item.setSavePath(NodePath.join(target.downloadDirectory, target.pendingDownloadGuid)); + target.pendingDownloadGuid = null; return true; }; @@ -418,6 +597,7 @@ export const make = Effect.gen(function* () { const id = keyOf(key); const tab = tabs.get(id); if (!tab) return; + for (const popup of tab.popups.values()) hidePopup(popup); tabs.delete(id); const announced = tab.stopWaiting === null; tab.stopWaiting?.(); @@ -432,44 +612,30 @@ export const make = Effect.gen(function* () { const id = keyOf(key); if (tabs.get(id)?.debuggee.webContents === debuggee.webContents) return; detach(key); - const tab: AttachedTab = { - key, - debuggee, - relay: null, - screenshotTurn: null, - waitingScreenshots: [], - downloadDirectory: null, - pendingDownloadGuid: null, - agentInputAt: Number.NEGATIVE_INFINITY, - focusEmulated: false, - focusChange: null, - focusUses: 0, - focusPending: new Map(), - painting: false, - agentUsed: false, - focusReleasing: false, - screencasts: new Set(), - onMessage: (_event, method, params, sessionId) => { - if (method === "Page.downloadWillBegin") { - const guid = (params as { guid?: unknown } | undefined)?.guid; - tab.pendingDownloadGuid = typeof guid === "string" ? guid : null; - } - tab.relay?.event(method, params, sessionId); - }, - // A screenshot pending when the page loads a new document, a reload included, can - // stay unanswered. A crashed page lands here once it reloads, well after its renderer died. - onNewDocument: () => { - // Before the next screenshot starts, so its own hold survives. - dropFocusHolds(tab); - if (tab.screenshotTurn !== null) nextScreenshot(tab); + const tab: AttachedTab = listen( + { + ...freshTarget(debuggee), + get tab() { + return tab; + }, + key, + relay: null, + owners: { detached: new Set(), streaming: new Set() }, + painting: false, + showsPopups: false, + popups: new Map(), + stopWaiting: null, }, - stopWaiting: null, - }; + () => tab.relay, + ); tabs.set(id, tab); debuggee.debugger.on("message", tab.onMessage); debuggee.webContents.on("did-navigate", tab.onNewDocument); const announce = () => { emit({ type: "attached", ...key }); + if (openedPopups.get(debuggee.webContents)?.size) { + emit({ type: "popupWindows", ...key, open: true }); + } const shown = shownTabs.get(id); if (shown !== undefined) emit({ type: "shown", ...key, shown }); }; @@ -534,22 +700,29 @@ export const make = Effect.gen(function* () { } const tab = tabs.get(keyOf(command.value)); if (!tab) return; - if (command.value.type === "pointer") { - const { threadId, tabId, phase, x, y } = command.value; - runFork(PubSub.publish(pointers, { key: { threadId, tabId }, phase, x, y })); - return; - } - if (command.value.type === "release") { - dropRelay(tab); - return; + switch (command.value.type) { + case "pointer": { + const { threadId, tabId, phase, x, y } = command.value; + runFork(PubSub.publish(pointers, { key: { threadId, tabId }, phase, x, y })); + return; + } + case "release": + dropRelay(tab); + return; + case "popups": + tab.showsPopups = true; + if (tab.relay) showPopups(tab); + return; + case "cdp": + relayFor(tab).receive(command.value.message); } - if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now(); - relayFor(tab).receive(command.value.message); }); const humanStartedDownload = (source: Electron.WebContents) => { - const tab = [...tabs.values()].find((candidate) => candidate.debuggee.webContents === source); - return tab !== undefined && performance.now() - tab.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS; + const target = targetOf(source); + return ( + target !== undefined && performance.now() - target.agentInputAt > AGENT_DOWNLOAD_WINDOW_MS + ); }; const announceAll = Effect.suspend(() => { @@ -563,6 +736,11 @@ export const make = Effect.gen(function* () { if (tab.stopWaiting !== null) return Effect.void; const shown = shownTabs.get(keyOf(tab.key)); return PubSub.publish(outbox, { type: "attached", ...tab.key }).pipe( + Effect.andThen( + openedPopups.get(tab.debuggee.webContents)?.size + ? PubSub.publish(outbox, { type: "popupWindows", ...tab.key, open: true }) + : Effect.void, + ), Effect.andThen( shown !== undefined ? PubSub.publish(outbox, { type: "shown", ...tab.key, shown }) @@ -588,6 +766,7 @@ export const make = Effect.gen(function* () { handleCommandLine, attach, detach, + openedPopup, setShown, placeDownload, humanStartedDownload, diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index b4e4b2a8b75ce..795f8aed4d8f5 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -614,6 +614,7 @@ describe("PreviewManager", () => { Layer.mock(DesktopBrowserHost.DesktopBrowserHost)({ attach: () => {}, detach: () => {}, + openedPopup: () => {}, setShown: () => {}, humanStartedDownload: () => true, placeDownload: () => true, diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index c668c39061e00..bce5262cb3308 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1460,6 +1460,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function window.webContents.on("before-input-event", (_event, input) => { syncMenuShortcuts(window.webContents, input); }); + browserHost.openedPopup(wc, window.webContents); }; const beforeInput = (event: Electron.Event, input: Electron.Input): void => { syncMenuShortcuts(wc, input); diff --git a/apps/server/src/preview/DesktopBrowserChannel.test.ts b/apps/server/src/preview/DesktopBrowserChannel.test.ts index ab5464cc4f6f7..8fb508198aab8 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.test.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.test.ts @@ -2,12 +2,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import * as NodeStream from "@effect/platform-node/NodeStream"; import { expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; +import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; +import { writeAllToFileDescriptor } from "../resourceTelemetry/DesktopTelemetryReceiver.ts"; import * as DesktopBrowserChannel from "./DesktopBrowserChannel.ts"; vi.mock("@effect/platform-node/NodeStream", () => ({ fromReadable: vi.fn() })); @@ -16,52 +20,115 @@ vi.mock("node:net", () => ({ destroy() {} }, })); +vi.mock("../resourceTelemetry/DesktopTelemetryReceiver.ts", () => ({ + writeAllToFileDescriptor: vi.fn(), +})); -it.effect("protects unknown desktop visibility until an explicit hidden report arrives", () => +const withChannel = ( + body: ( + channel: DesktopBrowserChannel.DesktopBrowserChannel["Service"], + send: (event: object) => Effect.Effect, + ) => Effect.Effect, +) => Effect.gen(function* () { const input = yield* Queue.unbounded(); vi.mocked(NodeStream.fromReadable).mockReturnValue(Stream.fromQueue(input)); const config = yield* ServerConfig.ServerConfig; - yield* Effect.gen(function* () { - const channel = yield* DesktopBrowserChannel.DesktopBrowserChannel; + const send = (event: object) => + Queue.offer(input, new TextEncoder().encode(`${JSON.stringify(event)}\n`)); + return yield* Effect.gen(function* () { + return yield* body(yield* DesktopBrowserChannel.DesktopBrowserChannel, send); + }).pipe( + Effect.provide( + DesktopBrowserChannel.layer.pipe( + Layer.provide( + ServerConfig.layer({ ...config, desktopBrowserFd: 41, desktopBrowserControlFd: 42 }), + ), + ), + ), + ); + }).pipe( + Effect.scoped, + Effect.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "desktop-browser-channel-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ), + ); + +it.effect("protects unknown desktop visibility until an explicit hidden report arrives", () => + withChannel((channel, send) => + Effect.gen(function* () { + vi.mocked(writeAllToFileDescriptor).mockReturnValue(Effect.void); const key = { threadId: "thread-1", tabId: "tab-1" }; - const send = (event: object) => - Queue.offer(input, new TextEncoder().encode(`${JSON.stringify(event)}\n`)); expect(channel.isShown(key)).toBe(true); yield* send({ type: "attached", ...key }); yield* channel.awaitAttached(key, "5 seconds"); expect(channel.isShown(key)).toBe(true); + expect(channel.hasNativePopups(key)).toBe(false); yield* send({ type: "shown", ...key, shown: false }); + yield* send({ type: "popupWindows", ...key, open: true }); yield* send({ type: "attached", threadId: key.threadId, tabId: "sentinel-1" }); yield* channel.awaitAttached({ threadId: key.threadId, tabId: "sentinel-1" }, "5 seconds"); expect(channel.isShown(key)).toBe(false); + expect(channel.hasNativePopups(key)).toBe(true); yield* send({ type: "shown", ...key, shown: true }); + yield* send({ type: "popupWindows", ...key, open: false }); yield* send({ type: "attached", threadId: key.threadId, tabId: "sentinel-2" }); yield* channel.awaitAttached({ threadId: key.threadId, tabId: "sentinel-2" }, "5 seconds"); expect(channel.isShown(key)).toBe(true); + expect(channel.hasNativePopups(key)).toBe(false); yield* send({ type: "shown", ...key, shown: false }); + yield* send({ type: "popupWindows", ...key, open: true }); yield* send({ type: "detached", ...key }); yield* send({ type: "attached", threadId: key.threadId, tabId: "sentinel-3" }); yield* channel.awaitAttached({ threadId: key.threadId, tabId: "sentinel-3" }, "5 seconds"); expect(channel.isShown(key)).toBe(true); - }).pipe( - Effect.provide( - DesktopBrowserChannel.layer.pipe( - Layer.provide( - ServerConfig.layer({ ...config, desktopBrowserFd: 41, desktopBrowserControlFd: 42 }), - ), - ), - ), - ); - }).pipe( - Effect.provide( - ServerConfig.layerTest(process.cwd(), { prefix: "desktop-browser-channel-" }).pipe( - Layer.provide(NodeServices.layer), - ), - ), + expect(channel.hasNativePopups(key)).toBe(false); + }), + ), +); + +it.live("a replaced endpoint's shutdown does not release the endpoint that replaced it", () => + withChannel((channel, send) => + Effect.gen(function* () { + const key = { threadId: "thread-1", tabId: "tab-1" }; + const commands = yield* Queue.unbounded<{ readonly type: string }>(); + vi.mocked(writeAllToFileDescriptor).mockImplementation((_fd, buffer) => + Queue.offer(commands, JSON.parse(buffer.toString())).pipe(Effect.asVoid), + ); + + yield* send({ type: "attached", ...key }); + expect(yield* channel.awaitAttached(key, "5 seconds")).toBe(true); + const first = yield* Scope.make(); + yield* channel.endpoint(key, { popups: true }).pipe(Scope.provide(first)); + expect((yield* Queue.take(commands)).type).toBe("popups"); + + // The desktop recovers a dropped debugger: the tab detaches and attaches again, and the + // reconnect opens its endpoint before the dropped connection finishes closing. + const detached = yield* channel.detached.pipe( + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* send({ type: "detached", ...key }); + yield* Fiber.join(detached); + yield* send({ type: "attached", ...key }); + expect(yield* channel.awaitAttached(key, "5 seconds")).toBe(true); + const second = yield* Scope.make(); + yield* channel.endpoint(key, { popups: true }).pipe(Scope.provide(second)); + expect((yield* Queue.take(commands)).type).toBe("popups"); + + yield* Scope.close(first, Exit.void); + yield* Scope.close(second, Exit.void); + // Commands are written in order, so everything the two closes sent precedes this one. + yield* channel.turn(key.threadId, false); + const sent: Array = []; + while (sent.at(-1) !== "turn") sent.push((yield* Queue.take(commands)).type); + expect(sent).toEqual(["release", "turn"]); + }), ), ); diff --git a/apps/server/src/preview/DesktopBrowserChannel.ts b/apps/server/src/preview/DesktopBrowserChannel.ts index 65129274206da..98067da5a81b9 100644 --- a/apps/server/src/preview/DesktopBrowserChannel.ts +++ b/apps/server/src/preview/DesktopBrowserChannel.ts @@ -46,7 +46,12 @@ export class DesktopBrowserChannel extends Context.Service< readonly isAttached: (key: DesktopTabKey) => Effect.Effect; /** Whether a desktop window draws the tab on screen, which no streamed viewer counts. */ readonly isShown: (key: DesktopTabKey) => boolean; - readonly endpoint: (key: DesktopTabKey) => Effect.Effect; + readonly hasNativePopups: (key: DesktopTabKey) => boolean; + /** With `popups`, the tab's popup windows become pages of the same connection. */ + readonly endpoint: ( + key: DesktopTabKey, + options?: { readonly popups: boolean }, + ) => Effect.Effect; readonly pointer: ( key: DesktopTabKey, pointer: { readonly phase: "move" | "click"; readonly x: number; readonly y: number }, @@ -64,6 +69,7 @@ const make = Effect.gen(function* () { const attachedTabs = new Set(); // Missing reports protect desktop tabs until the renderer explicitly marks them hidden. const shownTabs = new Map(); + const popupOpeners = new Set(); const inbound = new Map>(); const writeLock = yield* Semaphore.make(1); @@ -88,18 +94,16 @@ const make = Effect.gen(function* () { return Effect.void; }), isShown: () => false, + hasNativePopups: () => false, }); } - const command = (message: DesktopBrowserCommandType) => - writeLock.withPermits(1)( - encodeCommand(message).pipe( - Effect.flatMap((line) => writeAllToFileDescriptor(controlFd, Buffer.from(`${line}\n`))), - Effect.catchCause((cause) => - Effect.logWarning("desktop browser command failed", { cause }), - ), - ), + const write = (message: DesktopBrowserCommandType) => + encodeCommand(message).pipe( + Effect.flatMap((line) => writeAllToFileDescriptor(controlFd, Buffer.from(`${line}\n`))), + Effect.catchCause((cause) => Effect.logWarning("desktop browser command failed", { cause })), ); + const command = (message: DesktopBrowserCommandType) => writeLock.withPermits(1)(write(message)); // Use cancellable socket reads; filesystem pipe reads keep Node alive after scope shutdown. const readable = yield* Effect.acquireRelease( @@ -129,6 +133,7 @@ const make = Effect.gen(function* () { case "detached": { attachedTabs.delete(id); shownTabs.delete(id); + popupOpeners.delete(id); const queue = inbound.get(id); return (queue ? Queue.shutdown(queue) : Effect.void).pipe( Effect.andThen(PubSub.publish(changes, { key, attached: false })), @@ -137,14 +142,26 @@ const make = Effect.gen(function* () { case "shown": shownTabs.set(id, event.shown); return Effect.void; + case "popupWindows": + if (event.open) popupOpeners.add(id); + else popupOpeners.delete(id); + return Effect.void; } }), Effect.catchCause((cause) => Effect.logWarning("desktop browser channel stopped", { cause })), - Effect.ensuring(Effect.sync(() => shownTabs.clear())), + Effect.ensuring( + Effect.sync(() => { + shownTabs.clear(); + popupOpeners.clear(); + }), + ), Effect.forkScoped, ); - const endpoint = Effect.fn("DesktopBrowserChannel.endpoint")(function* (key: DesktopTabKey) { + const endpoint = Effect.fn("DesktopBrowserChannel.endpoint")(function* ( + key: DesktopTabKey, + options?: { readonly popups: boolean }, + ) { const id = keyOf(key); const secret = NodeCrypto.randomBytes(24).toString("base64url"); const server = yield* NodeSocketServer.makeWebSocket({ @@ -161,11 +178,19 @@ const make = Effect.gen(function* () { } yield* Effect.addFinalizer(() => Effect.gen(function* () { - if (inbound.get(id) === queue) inbound.delete(id); yield* Queue.shutdown(queue); - yield* command({ type: "release", ...key }); + // A reconnect's endpoint may already hold the tab; its release would drop the new relay. + yield* writeLock.withPermits(1)( + Effect.suspend(() => { + if (inbound.get(id) !== queue) return Effect.void; + inbound.delete(id); + return write({ type: "release", ...key }); + }), + ); }), ); + // Written before any CDP, so popups already open are there when Playwright connects. + if (options?.popups) yield* command({ type: "popups", ...key }); // The relay serves one Playwright connection; a second would see the first's sessions. let connected = false; yield* server @@ -232,6 +257,7 @@ const make = Effect.gen(function* () { return Effect.sync(() => attachedTabs.has(keyOf(key))); }), isShown: (key) => shownTabs.get(keyOf(key)) ?? true, + hasNativePopups: (key) => popupOpeners.has(keyOf(key)), endpoint, turn: Effect.fn("DesktopBrowserChannel.turn")(function (threadId, active) { return command({ type: "turn", threadId, active }); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index ff8442306f2ff..a3269192c3c94 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -35,6 +35,8 @@ export class PreviewManager extends Context.Service< readonly open: ( input: PreviewOpenInput & { readonly automationOwner?: string; + /** The desktop streams the tab from its first snapshot instead of hosting a webview. */ + readonly headless?: boolean; /** Runs before the `opened` event publishes, so subscribers find state keyed by the tab. */ readonly beforePublish?: (snapshot: PreviewSessionSnapshot) => void; }, @@ -42,7 +44,7 @@ export class PreviewManager extends Context.Service< readonly reportStatus: ( input: PreviewReportStatusInput & { readonly serverControlled?: boolean }, ) => Effect.Effect; - /** Records that a server tab runs headless. It stays headless until it closes. */ + /** Records that the desktop streams a server tab. It stays streamed until it closes. */ readonly markHeadless: ( input: PreviewCloseInput & { readonly tabId: string }, ) => Effect.Effect; @@ -197,6 +199,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { ? { automationOwner: input.automationOwner } : {}), ...(input.reveal === undefined ? {} : { reveal: input.reveal }), + ...(runtime === "server" && input.headless ? { headless: true } : {}), updatedAt, }; yield* SynchronizedRef.modifyEffect(stateRef, (state) => diff --git a/apps/server/src/preview/ServerBrowser.test.ts b/apps/server/src/preview/ServerBrowser.test.ts index c32947b30b995..118b76114ad10 100644 --- a/apps/server/src/preview/ServerBrowser.test.ts +++ b/apps/server/src/preview/ServerBrowser.test.ts @@ -59,6 +59,7 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ }; } contexts.push(context); + contextCreated?.resolve(context); return context as unknown as BrowserContext; } async scratchPage() { @@ -67,6 +68,9 @@ vi.mock("./ServerBrowserContexts.ts", () => ({ async connectDesktopPage(endpoint: string) { const context = makeContext(undefined, desktopPageUrl); context.page.viewportSize.mockReturnValue(null); + const popups = desktopPopupsNext.splice(0); + for (const popup of popups) popup.page.opener = async () => context.page; + context.pages = () => [context.page, ...popups.map((popup) => popup.page)]; desktopConnections.push({ endpoint, context }); return { browser: { close: async () => {} }, page: context.page as unknown as Page }; } @@ -161,6 +165,7 @@ function fileChooserElement(accept: string) { function makeContext(onClose?: (context: BrowserContext) => void, initialUrl = "about:blank") { const events = new NodeEvents.EventEmitter(); const nativeFileChooser = vi.fn(); + const dialogAutoDismissed = vi.fn(); const setChooserFiles = vi.fn(async (_files: unknown) => {}); const sessions: ReturnType[] = []; let url = initialUrl; @@ -175,6 +180,21 @@ function makeContext(onClose?: (context: BrowserContext) => void, initialUrl = " emit: (name: string, ...args: unknown[]) => events.emit(name, ...args), nativeFileChooser, setChooserFiles, + dialogAutoDismissed, + // Playwright dismisses a dialog nothing listens for before the page shows it. + openDialog: () => { + const dialog = { + type: () => "alert", + message: () => "hello", + defaultValue: () => "", + accept: vi.fn(async () => {}), + dismiss: vi.fn(async () => {}), + }; + if (!events.emit("dialog", dialog)) { + dialogAutoDismissed(); + } + return dialog; + }, openFileChooser: () => { if ( !events.emit("filechooser", { @@ -206,6 +226,7 @@ function makeContext(onClose?: (context: BrowserContext) => void, initialUrl = " initialNavigation = null; await pending?.(); }), + opener: async (): Promise => null, goBack: vi.fn(async () => {}), goForward: vi.fn(async () => {}), reload: vi.fn(async () => {}), @@ -235,6 +256,7 @@ function makeContext(onClose?: (context: BrowserContext) => void, initialUrl = " sessions, cssViewport, newPage: async () => page as unknown as Page, + pages: (): ReadonlyArray => [page], grantPermissions: vi.fn(async () => {}), newCDPSession: async (target: unknown) => { const session = makeSession( @@ -300,6 +322,7 @@ const contexts: ReturnType[] = []; const encoderStreams: Array<{ frameRate: number | undefined; bitsPerSecond: number }> = []; const contextRequests: Array<{ profileId: string; isolated: boolean }> = []; let contextGate: PromiseWithResolvers | null = null; +let contextCreated: PromiseWithResolvers> | null = null; let pageCopy = ""; let keyCopy = ""; let focusedFrame: ReturnType | null = null; @@ -308,6 +331,9 @@ let pageSetupFailure: Error | null = null; let initialNavigation: (() => Promise) | null = null; let desktopRendersNext = false; let desktopPageUrl = "about:blank"; +/** Popup windows already open when the next desktop page connects. */ +const desktopPopupsNext: Array> = []; +const desktopEndpoints: Array<{ tabId: string; popups: boolean | undefined }> = []; /** Replaces the answer to the desktop page's size reads. */ let desktopSizeAnswer: (() => Promise>) | null = null; const desktopDetaches = new NodeEvents.EventEmitter(); @@ -325,6 +351,7 @@ const desktopRenders = (tabId: string) => { let desktopAttachesLate = false; let desktopAttachWaiting = Promise.withResolvers(); const shownDesktopTabs = new Set(); +const nativePopupOpeners = new Set(); const releasedDesktopTabs: Array = []; const desktopTurns = new NodeEvents.EventEmitter(); const desktopConnections: Array<{ endpoint: string; context: ReturnType }> = []; @@ -407,13 +434,19 @@ const dependencies = Layer.mergeAll( ), isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)), isShown: (key) => shownDesktopTabs.has(key.tabId), - endpoint: (key) => - Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () => + hasNativePopups: (key) => nativePopupOpeners.has(key.tabId), + endpoint: (key, options) => + Effect.acquireRelease( Effect.sync(() => { - releasedDesktopTabs.push(key.tabId); - desktopDetaches.emit("release", key.tabId); - desktopReleased.resolve(); + desktopEndpoints.push({ tabId: key.tabId, popups: options?.popups }); + return `ws://desktop/${key.tabId}`; }), + () => + Effect.sync(() => { + releasedDesktopTabs.push(key.tabId); + desktopDetaches.emit("release", key.tabId); + desktopReleased.resolve(); + }), ), pointer: () => Effect.void, turn: (threadId, active) => Effect.sync(() => desktopTurns.emit("turn", threadId, active)), @@ -477,6 +510,7 @@ beforeEach(() => { encoderStreams.length = 0; contextRequests.length = 0; contextGate = null; + contextCreated = null; contextFailure = null; pageSetupFailure = null; initialNavigation = null; @@ -485,10 +519,13 @@ beforeEach(() => { focusedFrame = null; desktopTabs.clear(); shownDesktopTabs.clear(); + nativePopupOpeners.clear(); desktopRendersNext = false; desktopAttachesLate = false; desktopAttachWaiting = Promise.withResolvers(); desktopPageUrl = "about:blank"; + desktopPopupsNext.length = 0; + desktopEndpoints.length = 0; desktopSizeAnswer = null; desktopDetachReady = Promise.withResolvers(); desktopReleased = Promise.withResolvers(); @@ -655,6 +692,228 @@ it.effect("a tab the desktop does not draw is marked headless, so the desktop st ).pipe(Effect.provide(layer)), ); +it.effect("an agent's desktop tab's popup windows become its tabs, drawn by the desktop", () => + Effect.scoped( + Effect.gen(function* () { + const { broker } = yield* ready; + const manager = yield* Manager.PreviewManager; + const events = yield* manager.subscribeEvents; + const nextOpened = Effect.gen(function* () { + let event = yield* PubSub.take(events); + while (event.type !== "opened") event = yield* PubSub.take(events); + return event.snapshot; + }); + const existing = makeContext(undefined, "https://example.com/popout"); + desktopPopupsNext.push(existing); + desktopRendersNext = true; + const opener = yield* broker.invoke({ + scope, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + yield* nextOpened; + const existingTab = yield* nextOpened; + const later = makeContext(undefined, "https://example.com/auth"); + const openerPage = desktopConnections[0]!.context.page; + openerPage.emit("popup", later.page); + openerPage.emit("popup", existing.page); + const laterTab = yield* nextOpened; + + // Streamed from its first snapshot: the desktop never loads the popup in a webview of its own. + expect([existingTab.headless, laterTab.headless]).toEqual([true, true]); + expect(desktopEndpoints).toEqual([{ tabId: opener.tabId, popups: true }]); + const status = yield* broker.invoke({ + scope, + tabId: existingTab.tabId, + operation: "status", + input: {}, + }); + expect(status.tabs).toEqual( + expect.arrayContaining( + [existingTab, laterTab].map((popup) => + expect.objectContaining({ + tabId: popup.tabId, + openerTabId: opener.tabId, + ownedByCaller: true, + }), + ), + ), + ); + expect(status.tabs).toHaveLength(4); + // A native window keeps its own size and the person's file picker. + expect(existing.page.setViewportSize).not.toHaveBeenCalled(); + existing.page.openFileChooser(); + expect(existing.page.nativeFileChooser).toHaveBeenCalledOnce(); + + yield* broker.invoke({ scope, tabId: existingTab.tabId, operation: "close", input: {} }); + expect(existing.page.close).toHaveBeenCalled(); + expect(releasedDesktopTabs).toEqual([]); + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + expect(sessions.map((session) => session.tabId)).toContain(laterTab.tabId); + expect(sessions.map((session) => session.tabId)).not.toContain(existingTab.tabId); + + const failed = makeContext(undefined, "https://example.com/failed-setup"); + failed.newCDPSession = async () => { + throw new Error("Popup debugger setup failed"); + }; + openerPage.emit("popup", failed.page); + const failedTab = yield* nextOpened; + let closed = yield* PubSub.take(events); + while (closed.type !== "closed" || closed.tabId !== failedTab.tabId) + closed = yield* PubSub.take(events); + expect(failed.page.isClosed()).toBe(false); + expect(releasedDesktopTabs).toEqual([]); + + // Setup that fails after the tab could answer dialogs also leaves the person's window open. + const failedLate = makeContext(undefined, "https://example.com/failed-enable"); + const newLateSession = failedLate.newCDPSession; + failedLate.newCDPSession = async (target) => { + const session = await newLateSession(target); + const send = session.send.getMockImplementation()!; + session.send.mockImplementation(async (method, input) => { + if (method === "Page.enable") throw new Error("Popup debugger detached"); + return send(method, input); + }); + return session; + }; + openerPage.emit("popup", failedLate.page); + const failedLateTab = yield* nextOpened; + closed = yield* PubSub.take(events); + while (closed.type !== "closed" || closed.tabId !== failedLateTab.tabId) + closed = yield* PubSub.take(events); + expect(failedLate.page.isClosed()).toBe(false); + const afterFailure = yield* broker.invoke({ + scope, + tabId: opener.tabId!, + operation: "status", + input: {}, + }); + expect(afterFailure.tabs?.map((tab) => tab.tabId)).not.toContain(failedLateTab.tabId); + + // A dialog holding a popup's setup holds neither a client's close nor other tabs' events. + const answered = Promise.withResolvers(); + yield* Effect.addFinalizer(() => Effect.sync(() => answered.resolve())); + const holdSetUp = (url: string, holds: (method: string, input: unknown) => boolean) => { + const popup = makeContext(undefined, url); + const held = Promise.withResolvers(); + const reloaded = Promise.withResolvers(); + const newHeldSession = popup.newCDPSession; + popup.newCDPSession = async (target) => { + const session = await newHeldSession(target); + const send = session.send.getMockImplementation()!; + session.send.mockImplementation(async (method, input) => { + if (holds(method, input)) { + held.resolve(); + await answered.promise; + } + if (method === "Page.reload") reloaded.resolve(); + return send(method, input); + }); + return session; + }; + openerPage.emit("popup", popup.page); + const dialog = popup.page.openDialog(); + dialog.accept.mockImplementation(async () => answered.resolve()); + return { popup, dialog, held: held.promise, reloaded: reloaded.promise }; + }; + const heldProbe = holdSetUp( + "https://example.com/held-probe", + (method, input) => + method === "Runtime.evaluate" && + (input as { expression?: string }).expression === "navigator.pdfViewerEnabled", + ); + const heldProbeTab = yield* nextOpened; + const heldEnable = holdSetUp( + "https://example.com/held-enable", + (method) => method === "Page.enable", + ); + const heldEnableTab = yield* nextOpened; + yield* Effect.promise(() => Promise.all([heldProbe.held, heldEnable.held])); + const threadId = scope.thread.threadId; + for (const { tabId } of [heldProbeTab, heldEnableTab]) { + yield* manager.resize({ + threadId, + tabId, + viewport: { _tag: "freeform", width: 900, height: 700 }, + }); + yield* manager.adjust({ threadId, tabId, colorScheme: "dark" }); + yield* manager.adjust({ threadId, tabId, clear: "cache" }); + yield* manager.adjust({ threadId, tabId, hardReload: true }); + } + const closing = [heldProbe.popup.page, later.page].map( + (page) => new Promise((resolve) => page.once("close", () => resolve())), + ); + yield* manager.close({ threadId, tabId: heldProbeTab.tabId }); + yield* manager.close({ threadId, tabId: laterTab.tabId }); + yield* Effect.promise(() => Promise.all(closing)); + const whileHeld = yield* broker.invoke({ + scope, + tabId: opener.tabId!, + operation: "status", + input: {}, + }); + const listed = whileHeld.tabs?.map((tab) => tab.tabId); + expect(listed).toContain(heldEnableTab.tabId); + expect(listed).not.toContain(heldProbeTab.tabId); + expect(listed).not.toContain(laterTab.tabId); + expect(heldProbe.dialog.accept).not.toHaveBeenCalled(); + expect(heldProbe.dialog.dismiss).not.toHaveBeenCalled(); + + // Its setup refuses the closed tab once the dialog lets it finish. + const refused = Promise.withResolvers(); + const realClose = manager.close; + const closes = vi.spyOn(manager, "close").mockImplementation((input) => { + if (input.tabId === heldProbeTab.tabId) refused.resolve(); + return realClose(input); + }); + yield* Effect.addFinalizer(() => Effect.sync(() => closes.mockRestore())); + // A native window's dialog is the person's to answer there. + expect( + yield* broker + .invoke({ + scope, + tabId: heldEnableTab.tabId, + operation: "dialog", + input: { accept: true }, + }) + .pipe(Effect.flip), + ).toMatchObject({ reason: "humanControl" }); + heldEnable.popup.sessions[0]!.emit("Page.javascriptDialogClosed"); + answered.resolve(); + yield* Effect.promise(() => refused.promise); + const afterSetUp = yield* broker.invoke({ + scope, + tabId: opener.tabId!, + operation: "status", + input: {}, + }); + expect(afterSetUp.tabs?.map((tab) => tab.tabId)).not.toContain(heldProbeTab.tabId); + // The open popup's settings, queued behind its setup, apply in the order they were made. + yield* Effect.promise(() => heldEnable.reloaded); + const sent = heldEnable.popup.sessions[0]!.send.mock; + const order = (method: string) => + sent.invocationCallOrder[sent.calls.findIndex(([name]) => name === method)]!; + expect(heldEnable.popup.page.emulateMedia).toHaveBeenCalledWith({ colorScheme: "dark" }); + expect(heldEnable.popup.page.emulateMedia.mock.invocationCallOrder[0]).toBeLessThan( + order("Network.clearBrowserCache"), + ); + expect(order("Network.clearBrowserCache")).toBeLessThan(order("Page.reload")); + + // A person's desktop tab keeps its popup windows to itself. + desktopRendersNext = true; + const human = yield* manager.open({ threadId: scope.thread.threadId, runtime: "server" }); + yield* broker.invoke({ scope, tabId: human.tabId, operation: "status", input: {} }); + expect(desktopEndpoints.at(-1)).toEqual({ tabId: human.tabId, popups: false }); + + // A window that opens as its opener's tab closes stays the person's. + yield* broker.invoke({ scope, tabId: opener.tabId!, operation: "close", input: {} }); + const lateWindow = makeContext(undefined, "https://example.com/late"); + openerPage.emit("popup", lateWindow.page); + expect(lateWindow.page.close).not.toHaveBeenCalled(); + }), + ).pipe(Effect.provide(layer)), +); + it.effect("an agent's tab the reloaded desktop dropped goes back to the desktop", () => Effect.scoped( Effect.gen(function* () { @@ -912,10 +1171,9 @@ it.live("streams to a read-only viewer without allowing takeover, input, or view yield* viewer.input(message); expect(page.setViewportSize).not.toHaveBeenCalled(); expect(session.send.mock.calls.some(([method]) => method.startsWith("Input."))).toBe(false); - const outputs = yield* Queue.takeAll(viewer.output); - expect(outputs).toContainEqual( - expect.objectContaining({ _tag: "frame", data: Buffer.from("frame") }), - ); + const outputs = [yield* Queue.take(viewer.output)]; + while (outputs.at(-1)!._tag !== "frame") outputs.push(yield* Queue.take(viewer.output)); + expect(outputs.at(-1)).toMatchObject({ _tag: "frame", data: Buffer.from("frame") }); expect(outputs).toContainEqual(expect.objectContaining({ _tag: "viewport" })); expect(outputs).toContainEqual( expect.objectContaining({ _tag: "control", canOperate: false, controller: "agent" }), @@ -2290,111 +2548,288 @@ it.live("profiles reported by one client survive another client's report", () => ); it.live.each([ - { budget: "agent", limit: 8 }, - { budget: "server", limit: 32 }, -])("opens and popups reserve the $budget budget before session creation", ({ budget, limit }) => - Effect.scoped( - Effect.gen(function* () { - const { broker, browser, tabId } = yield* ready; - const manager = yield* Manager.PreviewManager; - const open = (target = scope) => - broker.invoke({ - scope: target, + { budget: "agent", limit: 8, host: "server", dialogState: "pending" }, + { budget: "server", limit: 32, host: "server", dialogState: "pending" }, + { budget: "agent", limit: 8, host: "desktop", dialogState: "pending" }, + { budget: "server", limit: 32, host: "desktop", dialogState: "pending" }, + { budget: "agent", limit: 8, host: "desktop", dialogState: "closedBeforeTab" }, + { budget: "agent", limit: 8, host: "desktop", dialogState: "closedDuringSetup" }, + { budget: "agent", limit: 8, host: "desktop", dialogState: "openedDuringSetup" }, + // A modal holds page commands until it is answered, as a native sheet does. + { budget: "agent", limit: 8, host: "desktop", dialogState: "holdsPdfProbe" }, + { budget: "server", limit: 32, host: "desktop", dialogState: "holdsPageEnable" }, +])( + "reserves the $budget budget for opens and $host popups ($dialogState dialog)", + ({ budget, limit, host, dialogState }) => + Effect.scoped( + Effect.gen(function* () { + desktopRendersNext = host === "desktop"; + const { broker, browser, tabId } = yield* ready; + const openerPage = + host === "desktop" ? desktopConnections[0]!.context.page : contexts[0]!.page; + const manager = yield* Manager.PreviewManager; + const open = (target = scope) => + broker.invoke({ + scope: target, + operation: "open", + input: { reuseExistingTab: false, show: false }, + }); + for (let index = 1; index < limit - 3; index++) { + yield* open(budget === "agent" ? scope : asSession(`other-${Math.floor(index / 8)}`)); + } + + const sessionGate = Promise.withResolvers(); + const launchGate = (contextGate = Promise.withResolvers()); + const sessionStarted = Promise.withResolvers(); + const realOpen = manager.open; + const delayedOpen = vi.spyOn(manager, "open").mockImplementation((input) => + Effect.promise(() => { + sessionStarted.resolve(); + return sessionGate.promise; + }).pipe(Effect.andThen(realOpen(input))), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessionGate.resolve(); + launchGate.resolve(); + delayedOpen.mockRestore(); + }), + ); + const events = yield* manager.subscribeEvents; + const first = yield* open().pipe(Effect.result, Effect.forkScoped); + yield* Effect.promise(() => sessionStarted.promise); + const popup = makeContext(); + const resized = Promise.withResolvers(); + const setViewportSize = popup.page.setViewportSize; + popup.page.setViewportSize = vi.fn(async (size) => { + await setViewportSize(size); + if (size?.width === 900) resized.resolve(); + }); + const newCDPSession = popup.newCDPSession; + let pendingDialog: ReturnType; + let changedDialog = false; + const answered = Promise.withResolvers(); + const held = Promise.withResolvers(); + const registered = Promise.withResolvers(); + const captureHeld = Promise.withResolvers(); + popup.page.waitForLoadState = vi.fn(async () => registered.resolve()); + const holding = dialogState.startsWith("holds"); + const holds = (method: string, input: unknown) => + dialogState === "holdsPdfProbe" + ? method === "Runtime.evaluate" && + (input as { expression?: string }).expression === "navigator.pdfViewerEnabled" + : dialogState === "holdsPageEnable" && method === "Page.enable"; + popup.newCDPSession = async (target) => { + await launchGate.promise; + const session = await newCDPSession(target); + const setUp = popup.sessions.length === 1; + const send = session.send.getMockImplementation()!; + session.send.mockImplementation(async (method, input) => { + if (setUp && holds(method, input)) { + held.resolve(); + await answered.promise; + } + // The modal holds every session's frames, so a viewer's first frame waits too. + if ( + !setUp && + holding && + (method === "Page.startScreencast" || method === "Page.captureScreenshot") + ) { + captureHeld.resolve(); + await answered.promise; + } + if (!changedDialog) { + if (method === "Runtime.evaluate" && dialogState === "closedBeforeTab") { + changedDialog = true; + session.emit("Page.javascriptDialogClosed"); + } + if (method === "Page.enable") { + changedDialog = true; + if (dialogState === "closedDuringSetup") + session.emit("Page.javascriptDialogClosed"); + if (dialogState === "openedDuringSetup") pendingDialog = popup.page.openDialog(); + } + } + return send(method, input); + }); + return session; + }; + openerPage.emit("popup", popup.page); + // A native window's dialog waits for the person while its tab is still being set up. + pendingDialog = popup.page.openDialog(); + pendingDialog.accept.mockImplementation(async () => answered.resolve()); + pendingDialog.dismiss.mockImplementation(async () => answered.resolve()); + expect(popup.page.dialogAutoDismissed).toHaveBeenCalledTimes(host === "desktop" ? 0 : 1); + const last = yield* open().pipe(Effect.result, Effect.forkScoped); + const overflow = yield* open().pipe(Effect.result, Effect.forkScoped); + // The host processes this after the opens, while manager creation is blocked. + const reused = yield* broker.invoke({ + scope, + tabId, operation: "open", - input: { reuseExistingTab: false, show: false }, + input: { show: false }, }); - for (let index = 1; index < limit - 3; index++) { - yield* open(budget === "agent" ? scope : asSession(`other-${Math.floor(index / 8)}`)); - } - - const sessionGate = Promise.withResolvers(); - const launchGate = (contextGate = Promise.withResolvers()); - const sessionStarted = Promise.withResolvers(); - const realOpen = manager.open; - const delayedOpen = vi.spyOn(manager, "open").mockImplementation((input) => - Effect.promise(() => { - sessionStarted.resolve(); - return sessionGate.promise; - }).pipe(Effect.andThen(realOpen(input))), - ); - yield* Effect.addFinalizer(() => - Effect.sync(() => { - sessionGate.resolve(); - launchGate.resolve(); - delayedOpen.mockRestore(); - }), - ); - const events = yield* manager.subscribeEvents; - const first = yield* open().pipe(Effect.result, Effect.forkScoped); - yield* Effect.promise(() => sessionStarted.promise); - const popup = makeContext(); - const resized = Promise.withResolvers(); - const setViewportSize = popup.page.setViewportSize; - popup.page.setViewportSize = vi.fn(async (size) => { - await setViewportSize(size); - if (size?.width === 900) resized.resolve(); - }); - const newCDPSession = popup.newCDPSession; - popup.newCDPSession = async (target) => { - await launchGate.promise; - return newCDPSession(target); - }; - contexts[0]!.page.emit("popup", popup.page); - const last = yield* open().pipe(Effect.result, Effect.forkScoped); - const overflow = yield* open().pipe(Effect.result, Effect.forkScoped); - // The host processes this after the opens, while manager creation is blocked. - const reused = yield* broker.invoke({ - scope, - tabId, - operation: "open", - input: { show: false }, - }); - expect(reused.tabId).toBe(tabId); - expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength( - limit - 3, - ); + expect(reused.tabId).toBe(tabId); + expect((yield* manager.list({ threadId: scope.thread.threadId })).sessions).toHaveLength( + limit - 3, + ); - sessionGate.resolve(); - for (let count = 0; count < 3; count++) { - let event = yield* PubSub.take(events); - while (event.type !== "opened") event = yield* PubSub.take(events); - yield* manager.resize({ - threadId: scope.thread.threadId, - tabId: event.tabId, - viewport: { _tag: "freeform", width: 900, height: 700 }, - }); - } - // The same slots must stay reserved after publication, while pages are still opening. - const duringLaunch = yield* open().pipe(Effect.result, Effect.forkScoped); - const rejectedPopup = makeContext(); - contexts[0]!.page.emit("popup", rejectedPopup.page); - yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); - launchGate.resolve(); - - expect((yield* Fiber.join(first))._tag).toBe("Success"); - expect((yield* Fiber.join(last))._tag).toBe("Success"); - for (const fiber of [overflow, duringLaunch]) { - expect(yield* Fiber.join(fiber)).toMatchObject({ - _tag: "Failure", - failure: { _tag: "PreviewAutomationControlInterruptedError", reason: "tabLimit" }, + sessionGate.resolve(); + for (let count = 0; count < 3; count++) { + let event = yield* PubSub.take(events); + while (event.type !== "opened") event = yield* PubSub.take(events); + yield* manager.resize({ + threadId: scope.thread.threadId, + tabId: event.tabId, + viewport: { _tag: "freeform", width: 900, height: 700 }, + }); + } + // The same slots must stay reserved after publication, while pages are still opening. + const duringLaunch = yield* open().pipe(Effect.result, Effect.forkScoped); + const rejectedPopup = makeContext(); + openerPage.emit("popup", rejectedPopup.page); + yield* broker.invoke({ scope, tabId, operation: "status", input: {} }); + launchGate.resolve(); + + expect((yield* Fiber.join(first))._tag).toBe("Success"); + expect((yield* Fiber.join(last))._tag).toBe("Success"); + for (const fiber of [overflow, duringLaunch]) { + expect(yield* Fiber.join(fiber)).toMatchObject({ + _tag: "Failure", + failure: { _tag: "PreviewAutomationControlInterruptedError", reason: "tabLimit" }, + }); + } + if (host === "server") yield* Effect.promise(() => resized.promise); + // A held setup waits for its dialog's answer; the others finish on their own. + if (host === "desktop") yield* Effect.promise(() => (holding ? held : registered).promise); + const status = yield* broker.invoke({ + scope, + tabId, + operation: "status", + input: {}, }); - } - yield* Effect.promise(() => resized.promise); - const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); - for (const session of sessions) - yield* browser.attachViewer(viewerInput(session.tabId, false)); - expect(sessions).toHaveLength(limit); - expect(popup.page.isClosed()).toBe(false); - expect(rejectedPopup.page.isClosed()).toBe(true); - const status = yield* broker.invoke({ - scope, - tabId, - operation: "status", - input: {}, - }); - expect(status.tabs).toHaveLength(limit); - }), - ).pipe(Effect.provide(layer)), + expect(status.tabs).toHaveLength(limit); + if (host === "desktop") { + const adopted = status.tabs!.find((tab) => tab.openerTabId === tabId)!; + const popupStatus = yield* broker.invoke({ + scope, + tabId: adopted.tabId, + operation: "status", + input: {}, + }); + if (dialogState.startsWith("closed")) { + expect(popupStatus.dialog).toBeNull(); + expect(pendingDialog.accept).not.toHaveBeenCalled(); + expect(pendingDialog.dismiss).not.toHaveBeenCalled(); + } else { + expect(popupStatus.dialog).toMatchObject({ type: "alert", message: "hello" }); + const accept = dialogState !== "holdsPageEnable"; + const answerAs = (target: typeof scope) => + broker.invoke({ + scope: target, + tabId: adopted.tabId, + operation: "dialog", + input: { accept }, + }); + const handoffs: Array> = []; + const personAnswers = dialogState === "holdsPdfProbe"; + let afterAnswer: Effect.Effect = Effect.void; + if (holding) { + // The held setup has not finished, yet only the tab's owner can answer its dialog. + expect(popup.page.waitForLoadState).not.toHaveBeenCalled(); + expect(yield* answerAs(asSession("intruder")).pipe(Effect.flip)).toMatchObject({ + reason: "agentMismatch", + }); + // A viewer that leaves while its first frame is held never captures again. + const leaving = yield* Scope.make(); + yield* browser + .attachViewer(viewerInput(adopted.tabId, true)) + .pipe(Effect.provideService(Scope.Scope, leaving)); + const leavingSession = popup.sessions.at(-1)!; + yield* Effect.promise(() => captureHeld.promise); + yield* Scope.close(leaving, Exit.void); + expect(leavingSession.detach).toHaveBeenCalledTimes(1); + // A fresh viewer gets control and can answer before its first frame. + const person = yield* browser.attachViewer(viewerInput(adopted.tabId, true)); + let framed = false; + const controlled = (byPerson: boolean) => + Effect.gen(function* () { + let control = yield* Queue.take(person.output); + while ( + control._tag !== "control" || + (control.controller === "you") !== byPerson + ) { + if (control._tag === "frame") framed = true; + control = yield* Queue.take(person.output); + } + }); + handoffs.push(yield* person.input({ type: "takeControl" }).pipe(Effect.forkScoped)); + yield* controlled(true); + expect(framed).toBe(false); + expect(yield* answerAs(scope).pipe(Effect.flip)).toMatchObject({ + reason: "humanControl", + }); + if (personAnswers) yield* person.input({ type: "dialog", accept }); + handoffs.push( + yield* person.input({ type: "releaseControl" }).pipe(Effect.forkScoped), + ); + yield* controlled(false); + expect(pendingDialog.accept).toHaveBeenCalledTimes(personAnswers ? 1 : 0); + expect(pendingDialog.dismiss).not.toHaveBeenCalled(); + afterAnswer = Effect.gen(function* () { + while (!framed) framed = (yield* Queue.take(person.output))._tag === "frame"; + const detachedAt = leavingSession.detach.mock.invocationCallOrder[0]!; + const { calls, invocationCallOrder } = leavingSession.send.mock; + expect( + calls.filter((_, index) => invocationCallOrder[index]! > detachedAt), + ).toEqual([]); + }); + // Its opener keeps working while the popup's setup is held. + expect( + yield* broker.invoke({ + scope, + tabId, + operation: "evaluate", + input: { expression: "document.title" }, + }), + ).toBe("evaluated"); + } + if (!personAnswers) { + // Electron would leave the native box open, so the agent cannot answer it. + expect(yield* answerAs(scope).pipe(Effect.flip)).toMatchObject({ + reason: "humanControl", + }); + // The person answers in the native window, which Playwright's own session reports. + popup.page.emit("dialogclosed", pendingDialog); + answered.resolve(); + const answeredStatus = yield* broker.invoke({ + scope, + tabId: adopted.tabId, + operation: "status", + input: {}, + }); + expect(answeredStatus.dialog).toBeNull(); + } + expect(pendingDialog.accept).toHaveBeenCalledTimes(personAnswers && accept ? 1 : 0); + expect(pendingDialog.dismiss).not.toHaveBeenCalled(); + if (holding) yield* Effect.promise(() => registered.promise); + for (const handoff of handoffs) yield* Fiber.join(handoff); + yield* afterAnswer; + } + } + const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); + for (const session of sessions) + yield* browser.attachViewer(viewerInput(session.tabId, false)); + expect(sessions).toHaveLength(limit); + expect(popup.page.isClosed()).toBe(false); + expect(rejectedPopup.page.isClosed()).toBe(host === "server"); + if (host === "desktop") { + // No tab answers the window it could not adopt, so its dialogs stay the person's. + rejectedPopup.page.openDialog(); + expect(rejectedPopup.page.dialogAutoDismissed).not.toHaveBeenCalled(); + } + }), + ).pipe(Effect.provide(layer)), ); it.live("failed session and browser launches release capacity, as does closing a tab", () => @@ -2453,6 +2888,30 @@ it.live("failed session and browser launches release capacity, as does closing a input: {}, }); expect(status.tabs).toHaveLength(8); + + // A person's tab closed while its browser launches is never listed or left open. + const launch = (contextGate = Promise.withResolvers()); + const created = (contextCreated = Promise.withResolvers>()); + yield* Effect.addFinalizer(() => Effect.sync(() => launch.resolve())); + const closedEarly = yield* manager.open({ + threadId: scope.thread.threadId, + runtime: "server", + }); + yield* manager.close({ threadId: scope.thread.threadId, tabId: closedEarly.tabId }); + launch.resolve(); + const { page } = yield* Effect.promise(() => created.promise); + yield* Effect.promise( + () => + new Promise((resolve) => + page.isClosed() ? resolve() : page.once("close", () => resolve()), + ), + ); + const afterClose = yield* broker.invoke({ + scope, + operation: "status", + input: {}, + }); + expect(afterClose.tabs?.map((tab) => tab.tabId)).not.toContain(closedEarly.tabId); }), ).pipe(Effect.provide(layer)), ); @@ -2474,11 +2933,31 @@ it.live("a full server takes back the slot of the longest idle agent tab nobody input: { reuseExistingTab: false, open: false }, }); }; + const popupEvents = yield* manager.subscribeEvents; + const popup = makeContext(undefined, "https://example.com/popout"); const opened: Array = []; - for (let index = 0; index < 32; index++) { + // The 32nd slot holds the native popup window of agent-3's first tab. + for (let index = 0; index < 31; index++) { now.mockReturnValue(index); + if (index === 24) desktopPopupsNext.push(popup); opened.push((yield* open(`agent-${Math.floor(index / 8)}`)).tabId!); } + let popupOpened = yield* PubSub.take(popupEvents); + while (popupOpened.type !== "opened" || !popupOpened.snapshot.headless) + popupOpened = yield* PubSub.take(popupEvents); + const popupTabId = popupOpened.snapshot.tabId; + yield* broker.invoke({ + scope: asSession("agent-3"), + operation: "status", + tabId: PreviewTabId.make(popupTabId), + input: {}, + }); + const refused = makeContext(undefined, "https://example.com/retained"); + desktopConnections[8]!.context.page.emit("popup", refused.page); + nativePopupOpeners.add(opened[24]!); + nativePopupOpeners.add(opened[8]!); + // DevTools can already hold a native window before Playwright ever sees its page. + nativePopupOpeners.add(opened[9]!); now.mockReturnValue(100); yield* broker.invoke({ @@ -2495,12 +2974,30 @@ it.live("a full server takes back the slot of the longest idle agent tab nobody now.mockReturnValue(31 * 60 * 1000); // A session at its own limit couldn't use a freed slot, so nothing closes for it. - expect(yield* open("agent-3").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(yield* open("agent-2").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); expect(releasedDesktopTabs).toEqual([]); // Tabs a desktop window shows keep their slots however long they sit idle. for (const tabId of opened) shownDesktopTabs.add(tabId); expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); expect(releasedDesktopTabs).toEqual([]); + // So does a hidden tab whose popup window is still open on the desktop. + shownDesktopTabs.delete(opened[24]!); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(releasedDesktopTabs).toEqual([]); + expect(popup.page.close).not.toHaveBeenCalled(); + shownDesktopTabs.add(opened[24]!); + shownDesktopTabs.delete(opened[8]!); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(releasedDesktopTabs).toEqual([]); + expect(refused.page.close).not.toHaveBeenCalled(); + // Debugger detachment closes Playwright's page, leaving the native window open. + refused.page.emit("close"); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(releasedDesktopTabs).toEqual([]); + shownDesktopTabs.add(opened[8]!); + shownDesktopTabs.delete(opened[9]!); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(releasedDesktopTabs).toEqual([]); shownDesktopTabs.clear(); shownDesktopTabs.add(opened[1]!); const events = yield* manager.subscribeEvents; @@ -2520,8 +3017,42 @@ it.live("a full server takes back the slot of the longest idle agent tab nobody expect(releasedDesktopTabs).toEqual([opened[2]]); const { sessions } = yield* manager.list({ threadId: scope.thread.threadId }); expect(sessions.map((session) => session.tabId).toSorted()).toEqual( - [...opened.filter((tabId) => tabId !== opened[2]), fresh.tabId!].toSorted(), + [...opened.filter((tabId) => tabId !== opened[2]), popupTabId, fresh.tabId!].toSorted(), ); + yield* Effect.promise(() => refused.page.close()); + nativePopupOpeners.delete(opened[8]!); + for (const session of sessions) shownDesktopTabs.add(session.tabId); + shownDesktopTabs.delete(opened[8]!); + yield* open("agent-new"); + expect(yield* Queue.take(releases)).toBe(opened[8]); + nativePopupOpeners.delete(opened[9]!); + for (const session of (yield* manager.list({ threadId: scope.thread.threadId })).sessions) + shownDesktopTabs.add(session.tabId); + shownDesktopTabs.delete(opened[9]!); + const started = Promise.withResolvers(); + const finish = Promise.withResolvers(); + desktopConnections[9]!.context.page.goto.mockImplementationOnce(async () => { + started.resolve(); + await finish.promise; + }); + yield* Effect.addFinalizer(() => Effect.sync(() => finish.resolve())); + const running = yield* broker + .invoke({ + scope: asSession("agent-1"), + tabId: PreviewTabId.make(opened[9]!), + operation: "navigate", + input: { url: "https://example.com/held" }, + }) + .pipe(Effect.forkScoped); + yield* Effect.promise(() => started.promise); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + expect(releasedDesktopTabs).not.toContain(opened[9]); + finish.resolve(); + yield* Fiber.join(running); + expect(yield* open("agent-new").pipe(Effect.flip)).toMatchObject({ reason: "tabLimit" }); + now.mockReturnValue(62 * 60 * 1000); + yield* open("agent-new"); + expect(yield* Queue.take(releases)).toBe(opened[9]); }), ).pipe(Effect.provide(layer)), ); diff --git a/apps/server/src/preview/ServerBrowser.ts b/apps/server/src/preview/ServerBrowser.ts index ee31b17cc5889..d7cf2b8cecff0 100644 --- a/apps/server/src/preview/ServerBrowser.ts +++ b/apps/server/src/preview/ServerBrowser.ts @@ -495,7 +495,17 @@ const make = Effect.gen(function* () { >(); const closedPendingTabs = new Set(); - const adoptedPages = new Map(); + const adoptedPages = new Map< + string, + { + readonly page: Page; + readonly openerTabId: string; + readonly native: boolean; + readonly pendingDialog?: { current: Dialog | null }; + } + >(); + /** A desktop popup can reach adoption from both its `popup` event and the connect sweep. */ + const adoptedPopups = new WeakSet(); const closedSessions = new Set(); let hostConnectionId: string | null = null; let viewerResizeOrder = 0; @@ -742,7 +752,8 @@ const make = Effect.gen(function* () { const sessionOpen = (tab: ServerTab) => !closedSessions.has(tabKey(tab.threadId, tab.tabId)); - const dropTab = (tab: ServerTab, closeSession: boolean) => { + /** `closePage: false` leaves a person's popup window open after its tab failed to adopt it. */ + const dropTab = (tab: ServerTab, closeSession: boolean, closePage = true) => { const key = tabKey(tab.threadId, tab.tabId); if (tabs.get(key) !== tab) return; tabs.delete(key); @@ -752,8 +763,10 @@ const make = Effect.gen(function* () { const end = tab.desktop && !closeSession && sessionOpen(tab) ? "reconnect" : "gone"; for (const viewer of tab.viewers) viewer.push({ _tag: end }); void tab.control.close().catch(constVoid); - if (tab.desktop) void tab.desktop.close().catch(constVoid); - else void tab.page.close().catch(constVoid); + if (closePage) { + if (tab.desktop) void tab.desktop.close().catch(constVoid); + else void tab.page.close().catch(constVoid); + } if (tab.isolatedContext) void tab.page.context().close().catch(constVoid); void tab.recording?.encoder.close().catch(constVoid); void NodeFSP.rm(downloadDir(tab), { recursive: true, force: true }).catch(constVoid); @@ -785,7 +798,10 @@ const make = Effect.gen(function* () { try { const endpoint = await runPromise( desktopChannel - .endpoint({ threadId: snapshot.threadId, tabId: snapshot.tabId }) + .endpoint( + { threadId: snapshot.threadId, tabId: snapshot.tabId }, + { popups: snapshot.automationOwner !== undefined }, + ) .pipe(Scope.provide(scope)), ); const connected = await contexts.connectDesktopPage(endpoint); @@ -819,13 +835,17 @@ const make = Effect.gen(function* () { ): Promise => { const adopted = adoptedPages.get(tabKey(snapshot.threadId, snapshot.tabId)); adoptedPages.delete(tabKey(snapshot.threadId, snapshot.tabId)); - const desktop = + const connection = adopted === undefined && (await desktopRenders(snapshot)) ? await connectDesktop(snapshot) : null; - if (desktop) opened.push(desktop.close); + // A desktop popup window shares its opener's connection, so closing it closes only its page. + const desktop = + connection ?? + (adopted?.native ? { page: adopted.page, close: () => adopted.page.close() } : null); + if (connection) opened.push(connection.close); // Otherwise a desktop that attaches late shows its own page, and the agent drives another. - if (!desktop && desktopChannel.available) { + if (!connection && desktopChannel.available) { await runPromise( manager .markHeadless({ threadId: ThreadId.make(snapshot.threadId), tabId: snapshot.tabId }) @@ -850,7 +870,9 @@ const make = Effect.gen(function* () { const page = adopted?.page ?? desktop?.page ?? (await context.newPage()); if (!desktop) opened.push(() => page.close()); const cdp = await context.newCDPSession(page); - await installPdfDocumentNavigation(cdp); + // A native popup can already show a modal, which holds page commands until it is answered. + const native = adopted?.native === true; + if (!native) await installPdfDocumentNavigation(cdp); // The desktop's page is already a normal browser. A popup's first request // went out before it became a tab; everything after presents as Chrome. if (!desktop) await presentAsChrome(cdp, host); @@ -887,7 +909,7 @@ const make = Effect.gen(function* () { openerTabId: adopted?.openerTabId, downloads: [], fileChooser: null, - dialog: null, + dialog: adopted?.pendingDialog?.current ?? null, setting: snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, colorScheme: "system", zoomFactor: 1, @@ -904,24 +926,63 @@ const make = Effect.gen(function* () { captureLock: Promise.resolve(), capturing: 0, }; - if (!desktop) { - await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); - } - await applyRendering(tab, snapshot); + page.on("dialog", (dialog) => { + tab.dialog = dialog; + broadcastControl(tab); + }); + // A person can answer a native window's dialog before this tab's own CDP session listens. + page.on("dialogclosed", (dialog) => { + if (tab.dialog !== dialog) return; + tab.dialog = null; + broadcastControl(tab); + }); if (desktop) { - // The desktop resizes and zooms its webview itself; viewers map input with the new size. - const remeasure = () => { - if (tab.viewers.size > 0) return broadcastViewport(tab).catch(constVoid); - }; - cdp.on("Page.frameResized", remeasure); - // A read a navigation broke has a fresh page to measure. - cdp.on("Page.loadEventFired", remeasure); - // Closing a dialog resizes nothing, and the host can close it without us. cdp.on("Page.javascriptDialogClosed", () => { tab.dialog = null; - return remeasure(); + if (tab.viewers.size > 0) return broadcastViewport(tab).catch(constVoid); + }); + } + const finishSetUp = async () => { + if (native) await installPdfDocumentNavigation(cdp); + if (!desktop) { + await page.setViewportSize(fixedViewportSize(tab.setting) ?? UNATTACHED_FILL_VIEWPORT); + } + await applyRendering(tab, snapshot); + if (desktop) { + // The desktop resizes and zooms its webview itself; viewers map input with the new size. + const remeasure = () => { + if (tab.viewers.size > 0) return broadcastViewport(tab).catch(constVoid); + }; + cdp.on("Page.frameResized", remeasure); + // A read a navigation broke has a fresh page to measure. + cdp.on("Page.loadEventFired", remeasure); + await cdp.send("Page.enable"); + } + }; + const key = tabKey(tab.threadId, tab.tabId); + const register = async () => { + if (closedPendingTabs.delete(key)) { + await control.close().catch(constVoid); + if (native) await page.close().catch(constVoid); + throw new ServerBrowserTabNotFoundError({ threadId: tab.threadId, tabId: tab.tabId }); + } + // Playwright cannot reload a crashed page, so it must leave the tab list. + page.on("close", () => dropTab(tab, true)); + page.on("crash", () => dropTab(tab, true)); + tabs.set(key, tab); + reportLiveTabs(); + }; + if (native) { + // Listed first so its owner can answer that modal; later actions queue behind the setup. + await register(); + await control.system(finishSetUp).catch((cause: unknown) => { + dropTab(tab, true, false); + throw cause; }); - await cdp.send("Page.enable"); + if (tab.closing) + throw new ServerBrowserTabNotFoundError({ threadId: tab.threadId, tabId: tab.tabId }); + } else { + await finishSetUp(); } const isMainNavigation = (request: { isNavigationRequest(): boolean; frame(): unknown }) => request.isNavigationRequest() && request.frame() === page.mainFrame(); @@ -953,7 +1014,7 @@ const make = Effect.gen(function* () { if (!desktop) void recordFrameCopies(tab, frame); }); // An adopted popup's frames may have loaded before the tab listened. - if (adopted) for (const frame of page.frames()) void recordFrameCopies(tab, frame); + if (adopted && !desktop) for (const frame of page.frames()) void recordFrameCopies(tab, frame); page.on("requestfailed", (request) => { const errorText = request.failure()?.errorText ?? ""; pushBounded(tab.networkEntries, { @@ -1010,26 +1071,22 @@ const make = Effect.gen(function* () { timestamp: new Date().toISOString(), }); }); - page.on("dialog", (dialog) => { - tab.dialog = dialog; - broadcastControl(tab); - }); page.on("download", (download) => void saveDownload(tab, download)); updateFileChooserInterception(tab); // Popups become tabs and keep `window.opener`, so sign-in popups can report back. page.on("popup", (popup) => void adoptPopup(tab, popup)); - // Playwright cannot reload a crashed page, so it must leave the tab list. - const key = tabKey(tab.threadId, tab.tabId); - if (closedPendingTabs.delete(key)) { - await control.close().catch(constVoid); - throw new ServerBrowserTabNotFoundError({ threadId: tab.threadId, tabId: tab.tabId }); - } - page.on("close", () => dropTab(tab, true)); - page.on("crash", () => dropTab(tab, true)); - tabs.set(key, tab); - reportLiveTabs(); + if (!native) await register(); // A desktop page navigates on its own and can finish loading before these listeners attach. if (desktop) void page.waitForLoadState("load").then(() => reportLoaded(tab), constVoid); + // Popups opened before the connection fired their `popup` event before the listener above. + if (connection) { + for (const candidate of page.context().pages()) { + if (candidate === page) continue; + void candidate.opener().then((opener) => { + if (opener === page) return adoptPopup(tab, candidate); + }, constVoid); + } + } // Existing desktop and popup PDFs can start loading before interception attaches. if ( snapshot.navStatus._tag === "Loading" && @@ -1344,10 +1401,24 @@ const make = Effect.gen(function* () { }; const adoptPopup = async (opener: ServerTab, popup: Page) => { - if (opener.closing) { - await popup.close().catch(constVoid); - return; + if (adoptedPopups.has(popup) || popup.isClosed()) return; + adoptedPopups.add(popup); + const native = opener.desktop !== null; + // A native window is the person's; only one the server opened closes with a refused adoption. + const refuse = async () => { + if (!native) await popup.close().catch(constVoid); + }; + const pendingDialog: { current: Dialog | null } = { current: null }; + if (native) { + // Keep native dialogs pending even when this popup never becomes an automation tab. + popup.on("dialog", (dialog) => { + pendingDialog.current = dialog; + }); + popup.on("dialogclosed", (dialog) => { + if (pendingDialog.current === dialog) pendingDialog.current = null; + }); } + if (opener.closing) return refuse(); const url = popup.url(); const controller = [...opener.viewers].find( @@ -1362,20 +1433,19 @@ const make = Effect.gen(function* () { ...(opener.control.agentId === null ? {} : { automationOwner: opener.control.agentId, reveal: false }), + // The popup's page already exists; a desktop hosting its own webview would load it again. + headless: desktopChannel.available, beforePublish: (snapshot) => adoptedPages.set(tabKey(snapshot.threadId, snapshot.tabId), { page: popup, openerTabId: opener.tabId, + native, + ...(native ? { pendingDialog } : {}), }), }), - ).then( - (tab) => { - if (opener.control.agentId === null) controller?.push({ _tag: "popup", tabId: tab.tabId }); - }, - async () => { - await popup.close().catch(constVoid); - }, - ); + ).then((tab) => { + if (opener.control.agentId === null) controller?.push({ _tag: "popup", tabId: tab.tabId }); + }, refuse); }; const ensureTab = (snapshot: PreviewSessionSnapshot): Promise => { @@ -1438,10 +1508,20 @@ const make = Effect.gen(function* () { const isVisible = (tab: ServerTab) => tab.viewers.size > 0 || (tab.desktop !== null && desktopChannel.isShown(tab)); + // A desktop popup is the person's own window, and closing its opener destroys it too. + const isNativePopup = (tab: ServerTab) => tab.desktop !== null && tab.openerTabId !== undefined; + const holdsNativePopup = (tab: ServerTab) => + isNativePopup(tab) || (tab.desktop !== null && desktopChannel.hasNativePopups(tab)); + const idleAgentTabs = () => { const cutoff = Date.now() - AGENT_TAB_IDLE_MS; return [...tabs.values()].filter( - (tab) => tab.control.agentId !== null && !isVisible(tab) && tab.usedAt < cutoff, + (tab) => + tab.control.agentId !== null && + !tab.agentOperating && + !isVisible(tab) && + tab.usedAt < cutoff && + !holdsNativePopup(tab), ); }; @@ -2165,6 +2245,12 @@ const make = Effect.gen(function* () { message: "A human controls this tab.", reason: "humanControl", }); + // Electron keeps a native window's dialog box open when CDP answers it, so the person must. + if (tab.dialog && isNativePopup(tab)) + throw new BrowserControlInterrupted({ + message: "This dialog waits for the person in its desktop window.", + reason: "humanControl", + }); await resolveDialog(tab, input as PreviewAutomationDialogInput); return statusWithTitle(tab, request.agentSessionId); } @@ -2332,44 +2418,67 @@ const make = Effect.gen(function* () { Effect.ignore, ); + // A dialog can hold a tab's setup for as long as a person leaves it open, so each tab's + // events wait only for that tab's earlier ones, and closing waits for none. + const tabEvents = new Map>(); + const afterTabEvents = (key: string, work: () => Promise) => { + const next = (tabEvents.get(key) ?? Promise.resolve()).then(work).then(constVoid, constVoid); + tabEvents.set(key, next); + void next.then(() => { + if (tabEvents.get(key) === next) tabEvents.delete(key); + }); + }; + const mirrorManagerEvent = (event: PreviewEvent) => - Effect.promise(async () => { - if (event.type === "opened" && event.snapshot.runtime === "server") { - const key = tabKey(event.snapshot.threadId, event.snapshot.tabId); - if (event.snapshot.automationOwner === undefined || pendingTabs.has(key) || tabs.has(key)) - await ensureTab(event.snapshot).catch(constVoid); + Effect.sync(() => { + const key = tabKey(event.threadId, event.tabId); + if (event.type === "opened") { + if (event.snapshot.runtime !== "server") return; + if (event.snapshot.automationOwner === undefined || pendingTabs.has(key) || tabs.has(key)) { + // Started now, so a later close finds it pending and it never registers. + const opening = ensureTab(event.snapshot); + afterTabEvents(key, () => opening); + } return; } - const key = tabKey(event.threadId, event.tabId); - const tab = tabs.get(key); - if (event.type === "closed" && !tab && pendingTabs.has(key)) closedPendingTabs.add(key); - if (!tab) return; if (event.type === "closed") { + const tab = tabs.get(key); + if (!tab) { + if (pendingTabs.has(key)) closedPendingTabs.add(key); + return; + } closedSessions.add(key); dropTab(tab, false); closedSessions.delete(key); return; } - if (event.type === "resized") { - await tab.control - .system(async () => { - if ( - !viewportSettingsEqual(event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, tab.setting) - ) { - await applySetting(tab, event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT); - } - await applyRendering(tab, event.snapshot); - const request = event.request; - if (request?.clear === "cookies") await tab.cdp.send("Network.clearBrowserCookies"); - if (request?.clear === "cache") await tab.cdp.send("Network.clearBrowserCache"); - if (request?.hardReload) await tab.cdp.send("Page.reload", { ignoreCache: true }); - }) - .catch((cause: unknown) => - runFork(Effect.logWarning("server preview could not apply a tab setting", { cause })), - ); - } + if (event.type === "resized") afterTabEvents(key, () => applyTabSetting(key, event)); }); + const applyTabSetting = async ( + key: string, + event: Extract, + ) => { + const tab = tabs.get(key); + if (!tab) return; + await tab.control + .system(async () => { + if (!viewportSettingsEqual(event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, tab.setting)) { + await applySetting(tab, event.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT); + } + await applyRendering(tab, event.snapshot); + const request = event.request; + if (request?.clear === "cookies") await tab.cdp.send("Network.clearBrowserCookies"); + if (request?.clear === "cache") await tab.cdp.send("Network.clearBrowserCache"); + if (request?.hardReload) await tab.cdp.send("Page.reload", { ignoreCache: true }); + }) + .catch((cause: unknown) => { + // Closing the tab ends the setting with it. + if (!tab.closing) + runFork(Effect.logWarning("server preview could not apply a tab setting", { cause })); + }); + }; + const releaseViewerInput = async (viewer: ViewerState, session: CDPSession) => { for (const { key, code } of viewer.pressedKeys.values()) { await session.send("Input.dispatchKeyEvent", { type: "keyUp", key, code }).catch(constVoid); @@ -2534,19 +2643,21 @@ const make = Effect.gen(function* () { capacity: VIEWER_OUTPUT_LIMIT, strategy: "dropping", }); + let detached = false; const session = yield* Effect.acquireRelease( Effect.tryPromise({ try: () => tab.page.context().newCDPSession(tab.page), catch: (cause) => new ServerBrowserLaunchError({ cause }), }), (session) => - Effect.promise(() => - session + Effect.promise(() => { + detached = true; + return session .send("Page.stopScreencast") .catch(constVoid) .then(() => session.detach()) - .catch(constVoid), - ), + .catch(constVoid); + }), ); const quality = Math.min(100, Math.max(1, Math.round(input.quality))); // Chromium may drop the final frame during a burst; send a still when it settles. @@ -2559,8 +2670,9 @@ const make = Effect.gen(function* () { const startScreencast = (scale: number) => { screencastScale = scale; screencastParams = screencastParams.then(async () => { - if (tab.capturing > 0) return; + if (detached || tab.capturing > 0) return; await session.send("Page.stopScreencast").catch(constVoid); + if (detached) return; await session .send("Page.startScreencast", { format: "jpeg", @@ -2629,8 +2741,8 @@ const make = Effect.gen(function* () { broadcastControl(tab); pushFileChooser(tab); const pushStill = async () => { - const data = await withCaptureLock(tab, () => - ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }), + const data = await withCaptureLock(tab, async () => + detached ? null : ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }), ).catch(() => null); if (data) viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void }); }; @@ -2686,10 +2798,15 @@ const make = Effect.gen(function* () { }); }); void broadcastViewport(tab).catch(constVoid); - yield* Effect.promise(() => startScreencast(1)); - // An idle page does not repaint for a new screencast, so the viewer - // starts from a still unless a live frame beat it. - if (!screencastStarted) yield* Effect.promise(pushStill); + // A page dialog can hold capture until it is answered, so the viewer + // returns first and can answer it. An idle page does not repaint for a + // new screencast, so the viewer starts from a still unless a live frame beat it. + yield* Effect.forkScoped( + Effect.promise(async () => { + await startScreencast(1); + if (!screencastStarted) await pushStill(); + }), + ); return { output, input: (raw: unknown) => diff --git a/packages/contracts/src/desktopBrowser.ts b/packages/contracts/src/desktopBrowser.ts index 52fb66885adfe..10f5463039441 100644 --- a/packages/contracts/src/desktopBrowser.ts +++ b/packages/contracts/src/desktopBrowser.ts @@ -14,12 +14,16 @@ export const DesktopBrowserEvent = Schema.Union([ Schema.Struct({ type: Schema.Literal("cdp"), ...TabKey, message: Schema.String }), /** Its `` started or stopped being on screen in a desktop window. */ Schema.Struct({ type: Schema.Literal("shown"), ...TabKey, shown: Schema.Boolean }), + /** Native window lifetime, independent of popup debugger sessions. */ + Schema.Struct({ type: Schema.Literal("popupWindows"), ...TabKey, open: Schema.Boolean }), ]); export type DesktopBrowserEvent = typeof DesktopBrowserEvent.Type; export const DesktopBrowserCommand = Schema.Union([ Schema.Struct({ type: Schema.Literal("cdp"), ...TabKey, message: Schema.String }), Schema.Struct({ type: Schema.Literal("release"), ...TabKey }), + /** The server drives this tab for an agent, so the popup windows it opens become tabs too. */ + Schema.Struct({ type: Schema.Literal("popups"), ...TabKey }), Schema.Struct({ type: Schema.Literal("pointer"), ...TabKey, diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index 1790a4797db29..7094c0ab66370 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -160,7 +160,10 @@ export const PreviewSessionSnapshot = Schema.Struct({ /** Fixed at open; Electron cannot change a webview partition after attach without losing page state. */ profileId: Schema.optional(BrowserProfileId), runtime: Schema.optional(PreviewRuntime), - /** The server runs this tab in headless Chromium, so the desktop streams it like other clients. */ + /** + * The desktop streams this tab like other clients: the server runs it in headless Chromium, + * or it is a popup window another desktop tab opened. + */ headless: Schema.optional(Schema.Boolean), automationOwner: Schema.optional(Schema.String), reveal: Schema.optional(Schema.Boolean),