From 7ed3bdaceeac2a143c94a3a535644badc8a12a0c Mon Sep 17 00:00:00 2001 From: Entrpi Date: Sun, 8 Nov 2026 15:03:23 +1100 Subject: [PATCH] fix(server): typing after a click by coordinates no longer hangs a preview tab With no locator, `type` found the focused element through `page.locator("*:focus").last()` and awaited its `count()`, which takes no timeout. That query runs in Playwright's utility world. On a desktop tab that lost its utility world while connecting, it never settles, so `type` hung and held the tab's SessionControl queue for good. Bounding count() alone would not help: `locator.evaluate` also resolves its target through `waitForSelector` in the utility world, so every `type` after a coordinate click would then fail after its timeout on exactly the tabs where agents must act by x and y. The focus check already ran in the page's main world, so the lookup now runs there too. The check moves to a module-level `focusTextTarget`. With no target it starts from `document.activeElement` and walks open shadow roots to the innermost focused element, as `.last()` did. It refuses iframe, frame, fencedframe, object and embed explicitly: from the top document it cannot confirm that a text field has focus inside a frame, and an editable frame would otherwise pass as contentEditable and get a false success. The no-locator path runs it with one `page.evaluate`, raced against a single expiry timer that throws a PreviewAutomationTimeoutError with `detail.timeoutMs` before any text is typed. (`page.evaluate` uses Playwright's UtilityScript, which is injected into the main context; it is not the utility world.) The locator path calls the same function and keeps its behaviour. Behaviour changes: - On a tab whose utility world never answers, `type` with no locator types. - On a page busy past `timeoutMs` it fails at `timeoutMs` without typing. Before, it waited past its timeout and then typed, or hung if the page never answered. - `document.activeElement` does not need the window to be focused, so an unfocused page can be accepted, as the locator path already allowed. - A navigation during the probe gives Playwright's execution-context error instead of not-editable; both happen before any typing. The timeout message "The page did not report its focused element within Xms, so no text was typed." reaches agents through the snapshot action timeline. The immediate tool result and all tool text are unchanged. Tests: two fake-page, fake-timer cases that hang on main (typing with a stalled locator count, and the expiry with a page that never answers), and a Chromium case that types into a shadow-root field after a click by x/y and refuses a field inside an editable frame. User story: As someone whose agent clicks a text field by its position in the preview, I want its typing to land in that field instead of freezing the preview tab. --- .../src/preview/ServerBrowserPage.test.ts | 68 +++++++++++ apps/server/src/preview/ServerBrowserPage.ts | 112 ++++++++++++------ 2 files changed, 141 insertions(+), 39 deletions(-) diff --git a/apps/server/src/preview/ServerBrowserPage.test.ts b/apps/server/src/preview/ServerBrowserPage.test.ts index 78f03f1f91b72..dcac6af693234 100644 --- a/apps/server/src/preview/ServerBrowserPage.test.ts +++ b/apps/server/src/preview/ServerBrowserPage.test.ts @@ -153,6 +153,25 @@ describe("server browser element refs", () => { expect(await page.getByRole("textbox", { name: "parent" }).inputValue()).toBe(""); }); + it("types after a click by coordinates into a shadow-root field, and refuses a field inside an editable frame", async () => { + await page.setContent( + `
`, + ); + const field = page.locator("#host #field"); + const box = (await field.boundingBox())!; + await page.mouse.click(box.x + box.width / 2, box.y + box.height / 2); + await ServerBrowserPage.type(page, { text: "shadow" }); + expect(await field.inputValue()).toBe("shadow"); + + // Refused for now: focus inside a frame cannot be checked from the top document. + const inner = page.frameLocator("iframe").locator("#inner"); + await inner.click(); + await expect(ServerBrowserPage.type(page, { text: "frame" })).rejects.toMatchObject({ + tag: "PreviewAutomationTargetNotEditableError", + }); + expect(await inner.inputValue()).toBe(""); + }); + it("shows dotted fields as dots wherever their value appears, keeping text values and refs", async () => { await page.setContent(`
password:
@@ -521,6 +540,55 @@ describe("server browser evaluate", () => { }); }); +describe("server browser type", () => { + // As on a desktop tab whose Playwright utility world never answers. + const stalledPage = (focusAnswer: () => Promise, typed: string[]) => + ({ + locator: () => ({ last: () => ({ count: () => new Promise(() => {}) }) }), + evaluate: focusAnswer, + keyboard: { insertText: async (text: string) => void typed.push(text) }, + }) as unknown as Page; + + it("types into the focused field when Playwright's utility world never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(async () => true, typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toBe("typed"); + expect(typed).toEqual(["hello"]); + } finally { + vi.useRealTimers(); + } + }); + + it("stops at its timeout without typing when the page never answers", async () => { + vi.useFakeTimers(); + try { + const typed: string[] = []; + let outcome: unknown = "pending"; + const page = stalledPage(() => new Promise(() => {}), typed); + void ServerBrowserPage.type(page, { text: "hello", timeoutMs: 300 }).then( + () => (outcome = "typed"), + (error: unknown) => (outcome = error), + ); + await vi.advanceTimersByTimeAsync(300); + expect(outcome).toMatchObject({ + tag: "PreviewAutomationTimeoutError", + detail: { timeoutMs: 300 }, + }); + expect(typed).toEqual([]); + } finally { + vi.useRealTimers(); + } + }); +}); + describe("server browser net errors", () => { it.each([ ["net::ERR_CONNECTION_REFUSED", "ERR_CONNECTION_REFUSED", -102], diff --git a/apps/server/src/preview/ServerBrowserPage.ts b/apps/server/src/preview/ServerBrowserPage.ts index 31cff821c6e42..b1c07670dc68a 100644 --- a/apps/server/src/preview/ServerBrowserPage.ts +++ b/apps/server/src/preview/ServerBrowserPage.ts @@ -733,6 +733,76 @@ const clickTarget = async ( return point; }; +type FocusCandidate = { + readonly tagName: string; + readonly type?: string; + readonly disabled?: boolean; + readonly readOnly?: boolean; + readonly isContentEditable?: boolean; + readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null; + readonly focus?: () => void; + getRootNode(): { readonly activeElement?: FocusCandidate | null }; + contains(other: FocusCandidate): boolean; +}; + +// Serialized into the page's main world, so it must not reference module scope. +const focusTextTarget = (target: FocusCandidate | null) => { + let element = + target ?? + (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document + .activeElement; + if (target === null) { + while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; + // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable. + if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName)) + return false; + } + if (element == null) return false; + // Require an enabled, focused text target to avoid losing input or sending it to the previous field. + const nonText = [ + "button", + "checkbox", + "color", + "file", + "hidden", + "image", + "radio", + "range", + "reset", + "submit", + ]; + const textControl = + element.tagName === "TEXTAREA" || + (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text")); + if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) { + return false; + } + element.focus?.(); + const active = element.getRootNode().activeElement; + return active != null && (active === element || element.contains(active)); +}; + +// A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get. +const focusedTextField = async (page: Page, timeout: number) => { + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new ServerBrowserOperationError({ + tag: "PreviewAutomationTimeoutError", + message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`, + detail: { timeoutMs: timeout }, + }), + ), + timeout, + ); + }); + return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() => + clearTimeout(timer), + ); +}; + export const type = async (page: Page, input: PreviewAutomationTypeInput) => { const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; const locator = targetLocator(page, input); @@ -740,46 +810,10 @@ export const type = async (page: Page, input: PreviewAutomationTypeInput) => { await locator.fill(input.text, { timeout }); return; } - // A shadow host also matches :focus; use its innermost focused descendant. - const target = locator ?? page.locator("*:focus").last(); const focused = - (locator !== null || (await target.count()) > 0) && - (await target.evaluate( - (element) => { - // Require an enabled, focused text target to avoid losing input or sending it to the previous field. - const control = element as unknown as { - readonly type?: string; - readonly disabled?: boolean; - readonly readOnly?: boolean; - readonly isContentEditable?: boolean; - readonly focus?: () => void; - }; - const nonText = [ - "button", - "checkbox", - "color", - "file", - "hidden", - "image", - "radio", - "range", - "reset", - "submit", - ]; - const textControl = - element.tagName === "TEXTAREA" || - (element.tagName === "INPUT" && !nonText.includes(control.type ?? "text")); - if (!(textControl || control.isContentEditable) || control.disabled || control.readOnly) { - return false; - } - control.focus?.(); - const root = element.getRootNode() as { readonly activeElement?: typeof element | null }; - const active = root.activeElement; - return active != null && (active === element || element.contains(active)); - }, - undefined, - { timeout }, - )); + locator === null + ? await focusedTextField(page, timeout) + : await locator.evaluate(focusTextTarget, undefined, { timeout }); if (focused !== true) { throw new ServerBrowserOperationError({ tag: "PreviewAutomationTargetNotEditableError",