feat(web): the browser's address bar searches, and agent typing no longer highlights the whole page

apps/desktop/src/preview/PreviewAutomation.ts

@@ -104,6 +104,11 @@ import {
 } from "./drawOverlayScript.ts";
 import * as PreviewScopes from "./PreviewScopes.ts";
 import { buildRevealElementScript } from "./revealElementScript.ts";
+import {
+  buildTextReceiverScript,
+  isTextReceiver,
+  type TextReceiver,
+} from "./textReceiverScript.ts";
 import {
   buildPickOverlayScript,
   PICK_OVERLAY_BINDING,
@@ -162,6 +167,18 @@ const SCREEN_LOCKED =
  */
 const PAGE_FOCUS_WAIT_MS = 250;
 const PAGE_FOCUS_POLL = "4 millis";
+/**
+ * How long typing waits for the click before it to put focus in a field. A
+ * page may focus its field a moment after the click (a search button that
+ * opens the box it then focuses). Together with the wait above this stays well
+ * inside the time the page holds the keyboard (PREVIEW_AGENT_KEYS_DEADLINE_MS).
+ */
+const TEXT_FIELD_WAIT_MS = 400;
+const TEXT_FIELD_POLL = "20 millis";
+/** How long a page has to select a field's contents before typing gives up. */
+const TEXT_FIELD_CLEAR_MS = 200;
+const NO_TEXT_FIELD =
+  "the click did not put the cursor in a field that takes text, so nothing was typed; aim at the text field itself, or press keys one at a time for a control like a date picker";
 /**
  * An agent's script may wait on purpose (a promise that settles after an
  * animation, a fetch), so it gets nearly all of the broker's deadline.
@@ -897,6 +914,15 @@ export const make = Effect.gen(function* PreviewAutomationMake() {
         method: string,
         params: Record<string, unknown>,
       ) => Effect.Effect<unknown, PreviewCommandError>,
+      page: {
+        readonly contents: WebContents;
+        /**
+         * The same check `send` makes, for a step that is not a CDP command:
+         * a reason not to touch the page now, or null. Ask in the same
+         * synchronous step as the act.
+         */
+        readonly refusal: () => string | null;
+      },
     ) => Effect.Effect<A, E, R>,
   ) =>
     Effect.suspend(() => {
@@ -924,30 +950,27 @@ export const make = Effect.gen(function* PreviewAutomationMake() {
               yield* Effect.sleep(PAGE_FOCUS_POLL);
             }
             let sent = 0;
+            const refusal = () => {
+              const tab = attached.get(webContentsId);
+              const reason =
+                controlEpoch !== undefined && tab !== undefined && tab.controlEpoch !== controlEpoch
+                  ? "the user took control of this tab"
+                  : contents.focusedFrame === null
+                    ? "the page did not have keyboard focus"
+                    : Date.now() - startedAt > PREVIEW_AGENT_KEYS_DEADLINE_MS
+                      ? "the page took too long to take the keys"
+                      : null;
+              if (reason === null) {
+                sent += 1;
+                return null;
+              }
+              return sent === 0
+                ? `${reason}, so nothing was typed; try again`
+                : `${reason}, so only part of the input reached it; check the field before trying again`;
+            };
             const send = (method: string, params: Record<string, unknown>) =>
-              sendCommand(contents, method, params, {
-                refusal: () => {
-                  const tab = attached.get(webContentsId);
-                  const refusal =
-                    controlEpoch !== undefined &&
-                    tab !== undefined &&
-                    tab.controlEpoch !== controlEpoch
-                      ? "the user took control of this tab"
-                      : contents.focusedFrame === null
-                        ? "the page did not have keyboard focus"
-                        : Date.now() - startedAt > PREVIEW_AGENT_KEYS_DEADLINE_MS
-                          ? "the page took too long to take the keys"
-                          : null;
-                  if (refusal === null) {
-                    sent += 1;
-                    return null;
-                  }
-                  return sent === 0
-                    ? `${refusal}, so nothing was typed; try again`
-                    : `${refusal}, so only part of the input reached it; check the field before trying again`;
-                },
-              });
-            return yield* keys(send);
+              sendCommand(contents, method, params, { refusal });
+            return yield* keys(send, { contents, refusal });
           }),
         ),
       );
@@ -956,6 +979,57 @@ export const make = Effect.gen(function* PreviewAutomationMake() {
   const failCommand = (contents: WebContents, method: string, cause: string) =>
     Effect.fail(new PreviewCommandError({ webContentsId: contents.id, method, cause }));
 
+  /** One run of the text-receiver script in the frame that holds the keyboard. */
+  const askTextReceiver = (contents: WebContents, script: string) =>
+    Effect.tryPromise(async (): Promise<unknown> =>
+      contents.focusedFrame?.executeJavaScript(script),
+    ).pipe(
+      Effect.map((answer): TextReceiver => (isTextReceiver(answer) ? answer : "none")),
+      // A page that cannot be asked has nothing to type into.
+      Effect.orElseSucceed((): TextReceiver => "none"),
+    );
+
+  /**
+   * What would take text typed now (`textReceiverScript.ts`), waiting a
+   * moment for the page to focus a field. Only looks; changes nothing.
+   */
+  const findTextReceiver = (contents: WebContents) =>
+    Effect.gen(function* () {
+      const script = buildTextReceiverScript();
+      for (;;) {
+        const receiver = yield* askTextReceiver(contents, script);
+        if (receiver !== "none") return receiver;
+        yield* Effect.sleep(TEXT_FIELD_POLL);
+      }
+    }).pipe(
+      Effect.timeoutOrElse({
+        duration: TEXT_FIELD_WAIT_MS,
+        orElse: () => Effect.succeed<TextReceiver>("none"),
+      }),
+    );
+
+  /**
+   * Selects a field's contents so the text typed next replaces them, in the
+   * same step in the page that checks the field still has focus, and says
+   * what it found there. The page is told when to give up, so one that
+   * answers late changes nothing.
+   */
+  const clearTextReceiver = (contents: WebContents, refusal: () => string | null) =>
+    Effect.suspend(() => {
+      const reason = refusal();
+      return reason === null
+        ? askTextReceiver(
+            contents,
+            buildTextReceiverScript({ clearBefore: Date.now() + TEXT_FIELD_CLEAR_MS }),
+          ).pipe(
+            Effect.timeoutOrElse({
+              duration: TEXT_FIELD_CLEAR_MS,
+              orElse: () => Effect.succeed<TextReceiver>("none"),
+            }),
+          )
+        : failCommand(contents, "Input.insertText", reason);
+    });
+
   const targetDescription = (target: PreviewAutomationTarget): string =>
     "ref" in target
       ? `ref ${target.ref}`
@@ -2843,28 +2917,31 @@ export const make = Effect.gen(function* PreviewAutomationMake() {
     // Into whatever the page has focused: the renderer clicks the field first,
     // as its own step, so the page needs the keyboard only for the keys.
     type: Effect.fn("PreviewAutomation.type")(function* (input: DesktopPreviewTypeInput) {
-      yield* agentKeys(input.webContentsId, input.controlEpoch, (send) =>
+      yield* agentKeys(input.webContentsId, input.controlEpoch, (send, { contents, refusal }) =>
         Effect.gen(function* () {
-          if (input.clear === true) {
-            // `commands` invokes the editing command directly, which is what a
-            // modifier chord ultimately triggers. Dispatching Meta+A as a raw key
-            // does not: the browser resolves shortcuts to editing commands above
-            // the layer CDP injects at, so the keypress arrives and selects
-            // nothing, and the insert below lands wherever the caret happened to
-            // be -- mid-word, since a click centres it.
-            yield* send("Input.dispatchKeyEvent", {
-              type: "keyDown",
-              key: "a",
-              code: "KeyA",
-              windowsVirtualKeyCode: 65,
-              commands: ["selectAll"],
-            });
-            yield* send("Input.dispatchKeyEvent", {
-              type: "keyUp",
+          // The click may have landed on something that takes no text: a
+          // button, a label, a layer over the field. Then nothing is sent,
+          // and the agent hears that nothing was typed. Sending "select all"
+          // regardless is what used to highlight the whole page.
+          let receiver = yield* findTextReceiver(contents);
+          if (receiver === "field" && input.clear === true) {
+            receiver = yield* clearTextReceiver(contents, refusal);
+          }
+          if (receiver === "none") {
+            return yield* failCommand(contents, "Input.insertText", NO_TEXT_FIELD);
+          }
+          if (receiver === "editContext" && input.clear === true) {
+            // The editor draws its own text and selects it on its own
+            // shortcut, so it gets the chord a person would press. No `text`:
+            // the chord must not also type an "a".
+            const chord = {
               key: "a",
               code: "KeyA",
               windowsVirtualKeyCode: 65,
-            });
+              modifiers: toCdpModifierBitmask([process.platform === "darwin" ? "Meta" : "Control"]),
+            };
+            yield* send("Input.dispatchKeyEvent", { type: "keyDown", ...chord });
+            yield* send("Input.dispatchKeyEvent", { type: "keyUp", ...chord });
           }
           yield* send("Input.insertText", { text: input.text });
           if (input.submit === true) {

apps/desktop/src/preview/textReceiverScript.test.ts

@@ -0,0 +1,15 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { buildTextReceiverScript } from "./textReceiverScript.ts";
+
+// The script is a string that only ever runs inside the guest page, where a
+// syntax error is invisible: typing would fail on every page with nothing to
+// say why. Parsing it here is what makes that loud.
+describe("buildTextReceiverScript", () => {
+  it("parses as JavaScript with and without clearing", () => {
+    expect(() => new Function(`return ${buildTextReceiverScript()}`)).not.toThrow();
+    expect(
+      () => new Function(`return ${buildTextReceiverScript({ clearBefore: 1 })}`),
+    ).not.toThrow();
+  });
+});

apps/desktop/src/preview/textReceiverScript.ts

@@ -0,0 +1,114 @@
+/**
+ * What in the page would take text an agent types now, asked in the frame
+ * that holds the keyboard.
+ *
+ * Typing clicks its target first and then sends text to whatever the page has
+ * focused. When the click lands on something that takes no text (a button, a
+ * label, a layer over the field), inserted text goes nowhere, and a "select
+ * all" sent to clear the field selects the whole page instead: the user sees
+ * every word on the page highlighted and the agent hears that it typed. So the
+ * receiver is found first, and clearing happens here, in the same step that
+ * checks it, and only inside it.
+ *
+ * - `field`: a text input, a textarea, rich text (contenteditable), or
+ *   anything else the page's own editor says takes text where its cursor is,
+ *   which covers a field inside a component whose insides cannot be read (a
+ *   closed shadow root).
+ * - `editContext`: an editor that draws its own text (EditContext, as newer
+ *   code editors do). It takes inserted text and selects on its own shortcut,
+ *   so it is not cleared here.
+ * - `none`: nothing that takes text. Date and time pickers are here too: they
+ *   look like inputs and ignore inserted text.
+ *
+ * Judged by the focused element, and by a cursor that is in or on it: a
+ * cursor left behind in a field the user typed in earlier would otherwise
+ * pass for focus. Known gap: an EditContext editor inside a closed shadow
+ * root reads as `none`. The script runs among the page's own scripts, so a
+ * page that lies to it only misleads typing on itself.
+ */
+export type TextReceiver = "field" | "editContext" | "none";
+
+/** Input types that take typed text. The rest are pickers, toggles and buttons. */
+const TEXT_INPUT_TYPES = ["text", "search", "url", "tel", "email", "password", "number"];
+
+export function isTextReceiver(value: unknown): value is TextReceiver {
+  return value === "field" || value === "editContext" || value === "none";
+}
+
+/**
+ * The script that names the receiver. With `clearBefore` (a time in epoch
+ * milliseconds) it also selects a `field`'s contents, so the text typed next
+ * replaces them, unless it only gets to run after that time: a page too busy
+ * to answer must not change its selection once the caller has given up.
+ */
+export function buildTextReceiverScript(options?: { readonly clearBefore: number }): string {
+  return `
+(() => {
+  const CLEAR_BEFORE = ${JSON.stringify(options?.clearBefore ?? null)};
+  const TEXT_INPUT_TYPES = ${JSON.stringify(TEXT_INPUT_TYPES)};
+  const clears = CLEAR_BEFORE !== null;
+  if (clears && Date.now() > CLEAR_BEFORE) {
+    return "none";
+  }
+
+  let focused = document.activeElement;
+  while (focused && focused.shadowRoot && focused.shadowRoot.activeElement) {
+    focused = focused.shadowRoot.activeElement;
+  }
+  if (!focused) {
+    return "none";
+  }
+  if (focused.editContext) {
+    return "editContext";
+  }
+
+  const tag = focused.localName;
+  if (tag === "input" || tag === "textarea") {
+    if (focused.disabled || focused.readOnly) {
+      return "none";
+    }
+    if (tag === "input" && !TEXT_INPUT_TYPES.includes(focused.type)) {
+      return "none";
+    }
+    if (clears) {
+      focused.select();
+    }
+    return "field";
+  }
+
+  const selection = getSelection();
+  const anchor = selection ? selection.anchorNode : null;
+  const cursorInside = anchor !== null && (anchor === focused || focused.contains(anchor));
+  if (!cursorInside) {
+    if (focused.isContentEditable) {
+      // Focused by script with the cursor left elsewhere: bring it in, or
+      // "select all" below would act on wherever it was.
+      if (clears) {
+        selection.selectAllChildren(focused);
+        selection.collapseToEnd();
+      }
+    } else {
+      // A field that cannot be read shows its cursor from outside as pointing
+      // at the element that holds it.
+      const pointedAt =
+        anchor !== null && anchor.nodeType === Node.ELEMENT_NODE
+          ? anchor.childNodes[selection.anchorOffset]
+          : null;
+      const cursorOnIt = pointedAt != null && (pointedAt === focused || pointedAt.contains(focused));
+      if (!cursorOnIt) {
+        return "none";
+      }
+    }
+  }
+  if (!focused.isContentEditable && !document.queryCommandEnabled("insertText")) {
+    return "none";
+  }
+  if (clears) {
+    // The browser's own "select all": with the cursor in a field it takes
+    // that field's contents, and editors that watch for it hear about it.
+    document.execCommand("selectAll");
+  }
+  return "field";
+})()
+`;
+}

apps/desktop/src/settings/DesktopClientSettings.test.ts

@@ -19,6 +19,7 @@ const clientSettings: ClientSettings = {
   agentBrowserProjectSitePolicy: {},
   agentBrowserTrustedCertificates: {},
   browserNewTabSize: null,
+  browserSearchEngine: "google",
   browserSignIns: "shared",
   autoOpenAgentsPanel: false,
   chatChangedFilesDefaultExpanded: false,

apps/server/src/guide/topics.ts

@@ -275,7 +275,9 @@ When a Claude or Codex usage limit stops a thread, the notice above the composer
 
 {place:general.browserSignIns} is Shared by default: every project on a computer uses one browser, so you sign in to a site once and every project's agents can use that sign-in. Local sites are shared too, so two projects on localhost:3000 see each other's cookies and storage. Per project gives each project its own. Each project still has its own allowed sites. The browser can't use Chrome's saved passwords or your Chrome sign-ins.
 
-Show device toolbar, in the browser's options menu (⋮), sets the page's size: Responsive fills the panel, and a Phone, Tablet, Laptop or Desktop size shrinks to fit it. New tabs open at {place:general.browserPageSize}, which the size menu can also set.`,
+Show device toolbar, in the browser's options menu (⋮), sets the page's size: Responsive fills the panel, and a Phone, Tablet, Laptop or Desktop size shrinks to fit it. New tabs open at {place:general.browserPageSize}, which the size menu can also set.
+
+Words typed in the browser's address bar are searched for, with the search engine set in {place:general.browserSearchEngine}. An address loads as it always did.`,
   },
   {
     id: "file-viewer",

apps/server/src/mcp/browserTools.ts

@@ -158,7 +158,7 @@ export const BrowserDragTool = changesThePage(
 export const BrowserTypeTool = changesThePage(
   Tool.make("browser_type", {
     description:
-      "Type into a field. Set `clear` to replace what is already there rather than appending, and `submit` to press Enter afterwards, which is how most single-field forms are sent.",
+      "Type into a field. Set `clear` to replace what is already there rather than appending, and `submit` to press Enter afterwards, which is how most single-field forms are sent. Name the text field itself: the field is clicked first, and when that click lands on something that takes no text (a button, a label, a date picker), nothing is typed and the call fails.",
     parameters: PreviewAutomationTypeInputSchema,
     success: PreviewAutomationActionStatusSchema,
     failure: PreviewAutomationErrorSchema,