diff --git a/.agents/skills/uloop-screenshot/references/annotated-elements.md b/.agents/skills/uloop-screenshot/references/annotated-elements.md index 5e1338e01..69466c096 100644 --- a/.agents/skills/uloop-screenshot/references/annotated-elements.md +++ b/.agents/skills/uloop-screenshot/references/annotated-elements.md @@ -10,7 +10,7 @@ Device Simulator is supported for this flow: prefer `--capture-mode rendering` ( - `Label`: Index label in JSON (`A` = first in that sort, `B` = next, ...). Treat it as a stable handle, not as proof of what is actually in front: when overlapping elements cross parents, pick the target by `Path` and confirm the result with `simulate-mouse-ui`'s `HitGameObjectName` — the handler that received the events, resolved up the hierarchy from the raycast hit (or from `--target-path` in bypass mode), so it can name an ancestor of the element you clicked. Screenshot labels also include the interaction hint, such as `A / CLICK` or `B / DRAG`. - `Name`: Element name -- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; see [UI Toolkit Elements](#ui-toolkit-elements). +- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; click them at `SimX`/`SimY` instead. See [UI Toolkit Elements](#ui-toolkit-elements). - `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`, `PhysicsCollider`) - `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`) or `Raycast` for clustered physics collider entries. Use this to choose between `simulate-mouse-ui --action Click`, drag actions, or `simulate-mouse-input` (including `--dry-run`). - `UiSystem`: UI system the entry came from: `UGUI`, `UIToolkit`, or empty for `PhysicsCollider` entries. @@ -53,9 +53,9 @@ Limits: Clicking: -- `simulate-mouse-input --action Click --x --y ` reaches UI Toolkit controls when the scene has an EventSystem with an Input System UI input module. -- Without an EventSystem the click does not reach UI Toolkit controls. -- `simulate-mouse-ui` does not reach UI Toolkit controls, and its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-ui --action Click --x --y ` (or `LongPress`) reaches UI Toolkit controls, with or without an EventSystem. Its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-input --action Click --x --y ` also reaches them when the scene has an EventSystem with an Input System UI input module. +- Without an EventSystem the `simulate-mouse-input` click does not reach UI Toolkit controls. ### PhysicsCollider Entries Per Closed Region diff --git a/.agents/skills/uloop-simulate-mouse-input/SKILL.md b/.agents/skills/uloop-simulate-mouse-input/SKILL.md index d7c92b3d4..41723aa25 100644 --- a/.agents/skills/uloop-simulate-mouse-input/SKILL.md +++ b/.agents/skills/uloop-simulate-mouse-input/SKILL.md @@ -67,7 +67,7 @@ All rows below assume the New Input System is installed. | Scenario | Tool | |----------|------| -| Click a Unity UI Button (IPointerClickHandler) | `simulate-mouse-ui` | +| Click a uGUI or UI Toolkit button | `simulate-mouse-ui` | | Runtime logic reads `Mouse.current.leftButton` | `simulate-mouse-input` | | Runtime logic reads right-click | `simulate-mouse-input --button Right` | | Drag a UI slider | `simulate-mouse-ui --action Drag` | @@ -110,7 +110,7 @@ uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --dura ## Coordinate System `--x`/`--y` use top-left Game View coordinates: pass `AnnotatedElements[].SimX/SimY` -directly (UI Toolkit entries need an EventSystem in the scene), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y +directly (UI Toolkit entries need an EventSystem with an Input System UI module here; `simulate-mouse-ui` reaches them without one), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y in the caller (the tool converts internally). Conversion formula, Device Simulator notes, and how the injected position reads back inside Unity are in `references/output-and-coordinates.md`. diff --git a/.agents/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md b/.agents/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md index a5998020b..979f721cb 100644 --- a/.agents/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md +++ b/.agents/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md @@ -30,7 +30,7 @@ Verify visual outcome with a follow-up screenshot. - `--x` / `--y` use **top-left Game View coordinates**. - Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. -- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. +- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. `simulate-mouse-ui` reaches them without an EventSystem. - Do not flip Y in the caller. The tool converts internally for Unity Input System: ```text diff --git a/.agents/skills/uloop-simulate-mouse-ui/SKILL.md b/.agents/skills/uloop-simulate-mouse-ui/SKILL.md index 03417c971..0be9c8c4f 100644 --- a/.agents/skills/uloop-simulate-mouse-ui/SKILL.md +++ b/.agents/skills/uloop-simulate-mouse-ui/SKILL.md @@ -1,7 +1,7 @@ --- name: uloop-simulate-mouse-ui toolName: simulate-mouse-ui -description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." +description: "Simulate PlayMode UI mouse actions on uGUI and UI Toolkit using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." --- # Task @@ -64,7 +64,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate -- Not for UI Toolkit entries (`UiSystem` `UIToolkit`); see `references/output.md`. +- UI Toolkit entries (`UiSystem` `UIToolkit`) can be clicked and long-pressed at `SimX`/`SimY`; `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObjects only. Details: `references/ui-toolkit.md`. - Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E) @@ -86,9 +86,6 @@ uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 # Long-press a button for 3 seconds uloop simulate-mouse-ui --action LongPress --x 400 --y 300 --duration 3.0 -# One-shot drag (start to end in one call) -uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 - # Slow drag for visual inspection uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --drag-speed 200 @@ -102,14 +99,14 @@ uloop simulate-mouse-ui --action DragEnd --x 600 --y 300 ## Prerequisites - Unity must be in **PlayMode** -- Target scene must have an **EventSystem** GameObject -- UI elements must have a **GraphicRaycaster** on their Canvas +- uGUI targets need an **EventSystem** and a **GraphicRaycaster** on their Canvas; UI Toolkit targets need neither - If you need runtime mouse input rather than UI pointer events, `simulate-mouse-input` assumes the project uses the New Input System; otherwise prefer `execute-dynamic-code` ## Output The response reports `Success`, `Message`, `Action`, `HitGameObjectName`, the used -coordinates, and `InterruptedByPausePoint`/`PausePointHits`. Two readings that matter: +coordinates, and `InterruptedByPausePoint`/`PausePointHits`; `UiSystem` and `HitPath` say +which UI system was hit and give its path in the annotation's form. Two readings that matter: `Click`/`LongPress` on empty space still return `Success = true` with `HitGameObjectName = null` (drag actions on empty space fail), and on a pause-point interruption `Message` states whether the pointer event fired before the pause. diff --git a/.agents/skills/uloop-simulate-mouse-ui/references/output.md b/.agents/skills/uloop-simulate-mouse-ui/references/output.md index 043b24d6a..af20d8c00 100644 --- a/.agents/skills/uloop-simulate-mouse-ui/references/output.md +++ b/.agents/skills/uloop-simulate-mouse-ui/references/output.md @@ -5,7 +5,9 @@ Returns JSON with: - `Success`: Whether the operation succeeded - `Message`: Status message (e.g. "Hit element: ButtonStart" or "No UI element under (x, y)") - `Action`: Echoes which action was executed (`Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, or `LongPress`) -- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself +- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself. For a UI Toolkit hit it is the control's `name`, the same as the annotation's `Name` +- `UiSystem`: Which UI system received the press (nullable string; `Click`/`LongPress` only, null if nothing was hit): `UGUI` for a target under a Canvas, `UIToolkit` for a UI Toolkit control, and `""` for a non-UI target such as a physics object +- `HitPath`: Path of the hit target in the form `AnnotatedElements[].Path` uses (nullable string; `Click`/`LongPress` only): a hierarchy path for GameObjects, `Document/Path#element` for UI Toolkit controls - `PositionX`: Target X coordinate that was used - `PositionY`: Target Y coordinate that was used - `EndPositionX`: Drag end X coordinate (nullable float; populated for drag actions only) @@ -18,4 +20,4 @@ Note: Click and LongPress on empty space (no UI element) still return `Success = ## UI Toolkit Elements -`simulate-mouse-ui` drives the uGUI EventSystem raycast path, so it does not reach UI Toolkit controls (`AnnotatedElements` entries with `UiSystem` `UIToolkit`): a click at their `SimX`/`SimY` lands on the panel's own GameObject instead. `--target-path` does not accept their `Path` either. Click them with `simulate-mouse-input` instead, which works when the scene has an EventSystem with an Input System UI input module. +`Click` and `LongPress` reach UI Toolkit controls at their annotated `SimX`/`SimY`. How presses are delivered and what is not supported: `ui-toolkit.md`. diff --git a/.agents/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md b/.agents/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md new file mode 100644 index 000000000..ec3bb79d4 --- /dev/null +++ b/.agents/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md @@ -0,0 +1,46 @@ +# simulate-mouse-ui and UI Toolkit + +`Click` and `LongPress` reach UI Toolkit controls on screen-space panels — the +`AnnotatedElements` entries with `UiSystem` `UIToolkit`. Pass the entry's `SimX`/`SimY` as +`--x`/`--y`; the response reports `UiSystem` `UIToolkit`, the control's `Name` as +`HitGameObjectName`, and its annotation `Path` as `HitPath`. + +## How a press is delivered + +- The point is mapped to each panel the same way the screenshot annotation maps controls to + `SimX`/`SimY`, so an annotated point lands on the annotated control. It does not depend on + which Editor view was drawn last. +- Panels are searched front to back by their panel settings' `sortingOrder`; the first panel + with a control under the point gets the press. The reported control is the outermost one an + annotation would list, so a click on a scrollbar's thumb reports the scrollbar. +- Where a uGUI element and a UI Toolkit panel are both under the point, the uGUI element wins + only when its Canvas sorts above the panel (raycaster sort priority greater than the panel's + `sortingOrder`), the same rule the EventSystem applies to real input. A uGUI image without + any pointer handler in front still blocks the panel, as real input would. +- A full-screen container left at the default `pickingMode` (`Position`) counts as a hit, so + uGUI underneath it is not reached — real input behaves the same way. +- UI Toolkit presses do not need an EventSystem. With neither an EventSystem nor a UI Toolkit + control under the point, the command fails and says so. +- A disabled control receives the press but does not react, as with real input. + +## LongPress + +A UI Toolkit `Button` fires `clicked` once when the long press is released (UI Toolkit's +`Clickable` behaviour); a uGUI long press sends no click. A `RepeatButton` fires repeatedly +while held. + +## Not supported + +- `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObject paths only; + a UI Toolkit `Path` (it contains `#`) is reported as not found. +- Drag actions go through the uGUI EventSystem only. +- World-space panels (Unity 6), as in the annotation. +- Two panels with the same `sortingOrder` overlapping at the point: Unity orders them by + creation order, which is not exposed, so either one may be chosen. + +## Notes + +- Move the real mouse cursor out of the Game View while simulating. Real pointer input over + the Game View reaches the same panels and can interleave with the simulated press. +- Device Simulator has the same limits as the annotation: coordinates use the simulated device + resolution. diff --git a/.claude/skills/uloop-screenshot/references/annotated-elements.md b/.claude/skills/uloop-screenshot/references/annotated-elements.md index 5e1338e01..69466c096 100644 --- a/.claude/skills/uloop-screenshot/references/annotated-elements.md +++ b/.claude/skills/uloop-screenshot/references/annotated-elements.md @@ -10,7 +10,7 @@ Device Simulator is supported for this flow: prefer `--capture-mode rendering` ( - `Label`: Index label in JSON (`A` = first in that sort, `B` = next, ...). Treat it as a stable handle, not as proof of what is actually in front: when overlapping elements cross parents, pick the target by `Path` and confirm the result with `simulate-mouse-ui`'s `HitGameObjectName` — the handler that received the events, resolved up the hierarchy from the raycast hit (or from `--target-path` in bypass mode), so it can name an ancestor of the element you clicked. Screenshot labels also include the interaction hint, such as `A / CLICK` or `B / DRAG`. - `Name`: Element name -- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; see [UI Toolkit Elements](#ui-toolkit-elements). +- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; click them at `SimX`/`SimY` instead. See [UI Toolkit Elements](#ui-toolkit-elements). - `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`, `PhysicsCollider`) - `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`) or `Raycast` for clustered physics collider entries. Use this to choose between `simulate-mouse-ui --action Click`, drag actions, or `simulate-mouse-input` (including `--dry-run`). - `UiSystem`: UI system the entry came from: `UGUI`, `UIToolkit`, or empty for `PhysicsCollider` entries. @@ -53,9 +53,9 @@ Limits: Clicking: -- `simulate-mouse-input --action Click --x --y ` reaches UI Toolkit controls when the scene has an EventSystem with an Input System UI input module. -- Without an EventSystem the click does not reach UI Toolkit controls. -- `simulate-mouse-ui` does not reach UI Toolkit controls, and its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-ui --action Click --x --y ` (or `LongPress`) reaches UI Toolkit controls, with or without an EventSystem. Its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-input --action Click --x --y ` also reaches them when the scene has an EventSystem with an Input System UI input module. +- Without an EventSystem the `simulate-mouse-input` click does not reach UI Toolkit controls. ### PhysicsCollider Entries Per Closed Region diff --git a/.claude/skills/uloop-simulate-mouse-input/SKILL.md b/.claude/skills/uloop-simulate-mouse-input/SKILL.md index d7c92b3d4..41723aa25 100644 --- a/.claude/skills/uloop-simulate-mouse-input/SKILL.md +++ b/.claude/skills/uloop-simulate-mouse-input/SKILL.md @@ -67,7 +67,7 @@ All rows below assume the New Input System is installed. | Scenario | Tool | |----------|------| -| Click a Unity UI Button (IPointerClickHandler) | `simulate-mouse-ui` | +| Click a uGUI or UI Toolkit button | `simulate-mouse-ui` | | Runtime logic reads `Mouse.current.leftButton` | `simulate-mouse-input` | | Runtime logic reads right-click | `simulate-mouse-input --button Right` | | Drag a UI slider | `simulate-mouse-ui --action Drag` | @@ -110,7 +110,7 @@ uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --dura ## Coordinate System `--x`/`--y` use top-left Game View coordinates: pass `AnnotatedElements[].SimX/SimY` -directly (UI Toolkit entries need an EventSystem in the scene), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y +directly (UI Toolkit entries need an EventSystem with an Input System UI module here; `simulate-mouse-ui` reaches them without one), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y in the caller (the tool converts internally). Conversion formula, Device Simulator notes, and how the injected position reads back inside Unity are in `references/output-and-coordinates.md`. diff --git a/.claude/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md b/.claude/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md index a5998020b..979f721cb 100644 --- a/.claude/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md +++ b/.claude/skills/uloop-simulate-mouse-input/references/output-and-coordinates.md @@ -30,7 +30,7 @@ Verify visual outcome with a follow-up screenshot. - `--x` / `--y` use **top-left Game View coordinates**. - Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. -- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. +- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. `simulate-mouse-ui` reaches them without an EventSystem. - Do not flip Y in the caller. The tool converts internally for Unity Input System: ```text diff --git a/.claude/skills/uloop-simulate-mouse-ui/SKILL.md b/.claude/skills/uloop-simulate-mouse-ui/SKILL.md index 03417c971..0be9c8c4f 100644 --- a/.claude/skills/uloop-simulate-mouse-ui/SKILL.md +++ b/.claude/skills/uloop-simulate-mouse-ui/SKILL.md @@ -1,7 +1,7 @@ --- name: uloop-simulate-mouse-ui toolName: simulate-mouse-ui -description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." +description: "Simulate PlayMode UI mouse actions on uGUI and UI Toolkit using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." --- # Task @@ -64,7 +64,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate -- Not for UI Toolkit entries (`UiSystem` `UIToolkit`); see `references/output.md`. +- UI Toolkit entries (`UiSystem` `UIToolkit`) can be clicked and long-pressed at `SimX`/`SimY`; `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObjects only. Details: `references/ui-toolkit.md`. - Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E) @@ -86,9 +86,6 @@ uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 # Long-press a button for 3 seconds uloop simulate-mouse-ui --action LongPress --x 400 --y 300 --duration 3.0 -# One-shot drag (start to end in one call) -uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 - # Slow drag for visual inspection uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --drag-speed 200 @@ -102,14 +99,14 @@ uloop simulate-mouse-ui --action DragEnd --x 600 --y 300 ## Prerequisites - Unity must be in **PlayMode** -- Target scene must have an **EventSystem** GameObject -- UI elements must have a **GraphicRaycaster** on their Canvas +- uGUI targets need an **EventSystem** and a **GraphicRaycaster** on their Canvas; UI Toolkit targets need neither - If you need runtime mouse input rather than UI pointer events, `simulate-mouse-input` assumes the project uses the New Input System; otherwise prefer `execute-dynamic-code` ## Output The response reports `Success`, `Message`, `Action`, `HitGameObjectName`, the used -coordinates, and `InterruptedByPausePoint`/`PausePointHits`. Two readings that matter: +coordinates, and `InterruptedByPausePoint`/`PausePointHits`; `UiSystem` and `HitPath` say +which UI system was hit and give its path in the annotation's form. Two readings that matter: `Click`/`LongPress` on empty space still return `Success = true` with `HitGameObjectName = null` (drag actions on empty space fail), and on a pause-point interruption `Message` states whether the pointer event fired before the pause. diff --git a/.claude/skills/uloop-simulate-mouse-ui/references/output.md b/.claude/skills/uloop-simulate-mouse-ui/references/output.md index 043b24d6a..af20d8c00 100644 --- a/.claude/skills/uloop-simulate-mouse-ui/references/output.md +++ b/.claude/skills/uloop-simulate-mouse-ui/references/output.md @@ -5,7 +5,9 @@ Returns JSON with: - `Success`: Whether the operation succeeded - `Message`: Status message (e.g. "Hit element: ButtonStart" or "No UI element under (x, y)") - `Action`: Echoes which action was executed (`Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, or `LongPress`) -- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself +- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself. For a UI Toolkit hit it is the control's `name`, the same as the annotation's `Name` +- `UiSystem`: Which UI system received the press (nullable string; `Click`/`LongPress` only, null if nothing was hit): `UGUI` for a target under a Canvas, `UIToolkit` for a UI Toolkit control, and `""` for a non-UI target such as a physics object +- `HitPath`: Path of the hit target in the form `AnnotatedElements[].Path` uses (nullable string; `Click`/`LongPress` only): a hierarchy path for GameObjects, `Document/Path#element` for UI Toolkit controls - `PositionX`: Target X coordinate that was used - `PositionY`: Target Y coordinate that was used - `EndPositionX`: Drag end X coordinate (nullable float; populated for drag actions only) @@ -18,4 +20,4 @@ Note: Click and LongPress on empty space (no UI element) still return `Success = ## UI Toolkit Elements -`simulate-mouse-ui` drives the uGUI EventSystem raycast path, so it does not reach UI Toolkit controls (`AnnotatedElements` entries with `UiSystem` `UIToolkit`): a click at their `SimX`/`SimY` lands on the panel's own GameObject instead. `--target-path` does not accept their `Path` either. Click them with `simulate-mouse-input` instead, which works when the scene has an EventSystem with an Input System UI input module. +`Click` and `LongPress` reach UI Toolkit controls at their annotated `SimX`/`SimY`. How presses are delivered and what is not supported: `ui-toolkit.md`. diff --git a/.claude/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md b/.claude/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md new file mode 100644 index 000000000..ec3bb79d4 --- /dev/null +++ b/.claude/skills/uloop-simulate-mouse-ui/references/ui-toolkit.md @@ -0,0 +1,46 @@ +# simulate-mouse-ui and UI Toolkit + +`Click` and `LongPress` reach UI Toolkit controls on screen-space panels — the +`AnnotatedElements` entries with `UiSystem` `UIToolkit`. Pass the entry's `SimX`/`SimY` as +`--x`/`--y`; the response reports `UiSystem` `UIToolkit`, the control's `Name` as +`HitGameObjectName`, and its annotation `Path` as `HitPath`. + +## How a press is delivered + +- The point is mapped to each panel the same way the screenshot annotation maps controls to + `SimX`/`SimY`, so an annotated point lands on the annotated control. It does not depend on + which Editor view was drawn last. +- Panels are searched front to back by their panel settings' `sortingOrder`; the first panel + with a control under the point gets the press. The reported control is the outermost one an + annotation would list, so a click on a scrollbar's thumb reports the scrollbar. +- Where a uGUI element and a UI Toolkit panel are both under the point, the uGUI element wins + only when its Canvas sorts above the panel (raycaster sort priority greater than the panel's + `sortingOrder`), the same rule the EventSystem applies to real input. A uGUI image without + any pointer handler in front still blocks the panel, as real input would. +- A full-screen container left at the default `pickingMode` (`Position`) counts as a hit, so + uGUI underneath it is not reached — real input behaves the same way. +- UI Toolkit presses do not need an EventSystem. With neither an EventSystem nor a UI Toolkit + control under the point, the command fails and says so. +- A disabled control receives the press but does not react, as with real input. + +## LongPress + +A UI Toolkit `Button` fires `clicked` once when the long press is released (UI Toolkit's +`Clickable` behaviour); a uGUI long press sends no click. A `RepeatButton` fires repeatedly +while held. + +## Not supported + +- `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObject paths only; + a UI Toolkit `Path` (it contains `#`) is reported as not found. +- Drag actions go through the uGUI EventSystem only. +- World-space panels (Unity 6), as in the annotation. +- Two panels with the same `sortingOrder` overlapping at the point: Unity orders them by + creation order, which is not exposed, so either one may be chosen. + +## Notes + +- Move the real mouse cursor out of the Game View while simulating. Real pointer input over + the Game View reaches the same panels and can interleave with the simulated press. +- Device Simulator has the same limits as the annotation: coordinates use the simulated device + resolution. diff --git a/Assets/Tests/Editor/MouseUiHitPriorityTests.cs b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs new file mode 100644 index 000000000..8120869ec --- /dev/null +++ b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs @@ -0,0 +1,64 @@ +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies which of an overlapping uGUI hit and UI Toolkit hit takes the input, following the EventSystem's + /// raycast ordering. + /// + public sealed class MouseUiHitPriorityTests + { + /// + /// Verifies a Canvas sorted above the panel puts uGUI in front. + /// + [Test] + public void IsUguiInFront_WhenCanvasSortsAbovePanel_ReturnsTrue() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(11, 10f), Is.True); + } + + /// + /// Verifies equal sort priorities leave UI Toolkit in front, since the panel's render order priority is + /// nearly int.MaxValue. + /// + [TestCase(10, 10f)] + [TestCase(10, 10.5f)] + [TestCase(-1, -0.5f)] + public void IsUguiInFront_WhenSortPrioritiesAreEqual_ReturnsFalse(int uguiSortOrderPriority, float panelSortingOrder) + { + Assert.That(MouseUiHitPriority.IsUguiInFront(uguiSortOrderPriority, panelSortingOrder), Is.False); + } + + /// + /// Verifies a fractional panel sorting order is rounded down, as PanelRaycaster does, so a Canvas just + /// above the rounded value comes in front; truncating or rounding up or to nearest would not. + /// + [TestCase(0, -0.5f)] + [TestCase(10, 9.5f)] + public void IsUguiInFront_WhenPanelSortingOrderIsFractional_RoundsDown(int uguiSortOrderPriority, float panelSortingOrder) + { + Assert.That(MouseUiHitPriority.IsUguiInFront(uguiSortOrderPriority, panelSortingOrder), Is.True); + } + + /// + /// Verifies a panel sorted above the Canvas stays in front. + /// + [Test] + public void IsUguiInFront_WhenPanelSortsAboveCanvas_ReturnsFalse() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(9, 10f), Is.False); + } + + /// + /// Verifies raycasters that keep the base priority, such as physics raycasters and non-overlay Canvases, + /// never come in front of a panel. + /// + [Test] + public void IsUguiInFront_WithBaseRaycasterPriority_ReturnsFalse() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(int.MinValue, -1000f), Is.False); + } + } +} diff --git a/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta new file mode 100644 index 000000000..f1d7b7580 --- /dev/null +++ b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 493c169e2fe5f485f956471f9840227a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs b/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs index d3281f00f..2f02091f9 100644 --- a/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs +++ b/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs @@ -71,6 +71,28 @@ public void ResolveGameObjectPath_WithUniquePath_ReturnsTarget() Assert.That(failureResponse, Is.Null); } + /// + /// Verifies a GameObject whose name contains '#' still resolves by its path, so the UI Toolkit hint for + /// '#' paths only applies when nothing matches. + /// + [Test] + public void ResolveGameObjectPath_WithHashInGameObjectName_ReturnsTarget() + { + GameObject root = CreateGameObject("MouseUiTargetResolverTests_HashRoot"); + GameObject expectedTarget = CreateGameObject("Menu#Play", root.transform); + string targetPath = GameObjectPathUtility.GetFullPath(expectedTarget); + + (GameObject? target, SimulateMouseUiResponse? failureResponse) = + MouseUiPointerTargetResolver.ResolveGameObjectPath( + targetPath, + "TargetPath", + MouseAction.Click, + new Vector2(10f, 20f)); + + Assert.That(target, Is.SameAs(expectedTarget)); + Assert.That(failureResponse, Is.Null); + } + /// /// Verifies a missing hierarchy path returns the exact wire-visible failure response. /// diff --git a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs index f7f9ab351..f09de75b6 100644 --- a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs +++ b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs @@ -103,7 +103,7 @@ public void CreateClickResult_WithHitMode_MapsMessageAndFields( Vector2 position = new(10.25f, 20.75f); SimulateMouseUiResponse response = - MouseUiSimulationResponseFactory.CreateClickResult(command, position, targetName, hitTarget); + MouseUiSimulationResponseFactory.CreateClickResult(command, position, CreateHit(hitTarget, targetName)); Assert.That(response.Success, Is.True); Assert.That(response.Message, Is.EqualTo(expectedMessage)); @@ -136,7 +136,7 @@ public void CreateLongPressResult_WithHitMode_MapsMessageAndFields( Vector2 position = new(10.25f, 20.75f); SimulateMouseUiResponse response = - MouseUiSimulationResponseFactory.CreateLongPressResult(command, position, targetName, hitTarget); + MouseUiSimulationResponseFactory.CreateLongPressResult(command, position, CreateHit(hitTarget, targetName)); Assert.That(response.Success, Is.True); Assert.That(response.Message, Is.EqualTo(expectedMessage)); @@ -146,6 +146,52 @@ public void CreateLongPressResult_WithHitMode_MapsMessageAndFields( Assert.That(response.PositionY, Is.EqualTo(position.y)); } + /// + /// Verifies click and long-press results carry the hit's UI system and path, and name UI Toolkit hits as + /// such in the message. + /// + [Test] + public void CreatePressResults_WithUIToolkitHit_MapUiSystemAndPath() + { + MouseUiHitReport hit = new MouseUiHitReport("play", "Menu/Document#play", "UIToolkit"); + Vector2 position = new(200f, 125f); + + SimulateMouseUiResponse click = MouseUiSimulationResponseFactory.CreateClickResult( + CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.Click }), position, hit); + SimulateMouseUiResponse longPress = MouseUiSimulationResponseFactory.CreateLongPressResult( + CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.LongPress, Duration = 0.5f }), position, hit); + + Assert.That(click.Message, Is.EqualTo("Clicked UI Toolkit element 'play' at (200.0, 125.0)")); + Assert.That(longPress.Message, Is.EqualTo("Long-pressed UI Toolkit element 'play' at (200.0, 125.0) for 0.5s")); + foreach (SimulateMouseUiResponse response in new[] { click, longPress }) + { + Assert.That(response.HitGameObjectName, Is.EqualTo("play")); + Assert.That(response.UiSystem, Is.EqualTo("UIToolkit")); + Assert.That(response.HitPath, Is.EqualTo("Menu/Document#play")); + } + } + + /// + /// Verifies a uGUI hit keeps its message and carries its UI system and path, and no hit leaves both null. + /// + [Test] + public void CreateClickResult_WithUguiHitOrNoHit_MapsUiSystemAndPath() + { + MouseUiSimulationCommand command = CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.Click }); + Vector2 position = new(10f, 20f); + + SimulateMouseUiResponse ugui = MouseUiSimulationResponseFactory.CreateClickResult( + command, position, new MouseUiHitReport("Ok", "Canvas/Ok", "UGUI")); + SimulateMouseUiResponse none = MouseUiSimulationResponseFactory.CreateClickResult( + command, position, MouseUiHitReport.None); + + Assert.That(ugui.Message, Is.EqualTo("Clicked 'Ok' at (10.0, 20.0)")); + Assert.That(ugui.UiSystem, Is.EqualTo("UGUI")); + Assert.That(ugui.HitPath, Is.EqualTo("Canvas/Ok")); + Assert.That(none.UiSystem, Is.Null); + Assert.That(none.HitPath, Is.Null); + } + /// /// Verifies one-shot drag results retain normal and bypass message contracts. /// @@ -218,5 +264,10 @@ private static MouseUiSimulationCommand CreateCommand(SimulateMouseUiSchema sche Assert.That(command, Is.Not.Null); return command!; } + + private static MouseUiHitReport CreateHit(bool hitTarget, string? targetName) + { + return hitTarget ? new MouseUiHitReport(targetName, null, null) : MouseUiHitReport.None; + } } } diff --git a/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs b/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs index bbc231a34..56e2406c9 100644 --- a/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs +++ b/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs @@ -355,14 +355,19 @@ public void ExecuteDynamicCodeUseCase_WhenAwaitingWorkflow_DoesNotCaptureEditorS [Test] public void MouseUiPressActionExecutor_WhenReturningAfterTimeout_UsesCapturedUnityObjectState() { - // Tests that timeout result branches use plain captured bools instead of UnityEngine.Object null checks. + // Tests that timeout result branches use the hit report captured before the first await, and that + // the release decision is a captured bool, instead of UnityEngine.Object null checks after awaits. string source = ReadSourceFile( "Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs"); - - Assert.That(source, Does.Contain("bool hitTarget = resolvedTargets.Target != null;")); - Assert.That(source, Does.Contain("bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null;")); - Assert.That(source, Does.Not.Contain("CreateClickResult(parameters, inputPos, targetName, resolvedTargets.Target != null)")); - Assert.That(source, Does.Not.Contain("CreateLongPressResult(parameters, inputPos, targetName, resolvedTargets.Target != null)")); + string uguiTargetSource = ReadSourceFile( + "Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs"); + + Assert.That(source, Does.Contain("MouseUiHitReport report = pressTarget.Report;")); + Assert.That(uguiTargetSource, Does.Contain("bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null;")); + Assert.That(uguiTargetSource, Does.Contain("if (!_shouldReleasePointer)")); + Assert.That(source, Does.Not.Contain("CreateClickResult(parameters, inputPos, pressTarget.Report)")); + Assert.That(source, Does.Not.Contain("CreateLongPressResult(parameters, inputPos, pressTarget.Report)")); + Assert.That(source, Does.Not.Contain("resolvedTargets.Target")); } [Test] diff --git a/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs new file mode 100644 index 000000000..aa6714045 --- /dev/null +++ b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.EventSystems; +using UnityEngine.UI; +using UnityEngine.UIElements; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies how EventSystem results that come from uGUI's UI Toolkit bridge are handled when choosing a hit. + /// The results are built by hand with real raycaster components as their module, because in EditMode the + /// bridge raycaster never registers and returns nothing. No Canvas-space hit is passed, since comparing one + /// reads Screen. + /// + public sealed class UiRaycastHelperPanelSkipTests + { + private readonly List _createdObjects = new List(); + + [TearDown] + public void TearDown() + { + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.DestroyImmediate(createdObject); + } + } + + _createdObjects.Clear(); + } + + /// + /// Verifies a bridge hit in first place is skipped and the next hit is chosen. + /// + [Test] + public void ChooseHitSkippingUIToolkitPanels_WhenFirstHitIsPanelRaycaster_ReturnsNextHit() + { + RaycastResult panelHit = CreatePanelHit(); + RaycastResult graphicHit = CreateGraphicHit(); + + RaycastResult? chosen = UiRaycastHelper.ChooseHitSkippingUIToolkitPanels( + new List { panelHit, graphicHit }, null); + + Assert.That(chosen.HasValue, Is.True); + Assert.That(chosen.Value.gameObject, Is.SameAs(graphicHit.gameObject)); + } + + /// + /// Verifies only bridge hits leave nothing to choose. + /// + [Test] + public void ChooseHitSkippingUIToolkitPanels_WhenOnlyPanelRaycaster_ReturnsNull() + { + RaycastResult? chosen = UiRaycastHelper.ChooseHitSkippingUIToolkitPanels( + new List { CreatePanelHit() }, null); + + Assert.That(chosen.HasValue, Is.False); + } + + /// + /// Verifies the unfiltered choice still returns a bridge hit in first place, so other callers keep their + /// current results. + /// + [Test] + public void ChooseHit_WhenFirstHitIsPanelRaycaster_ReturnsIt() + { + RaycastResult panelHit = CreatePanelHit(); + + RaycastResult? chosen = UiRaycastHelper.ChooseHit( + new List { panelHit, CreateGraphicHit() }, null); + + Assert.That(chosen.HasValue, Is.True); + Assert.That(chosen.Value.gameObject, Is.SameAs(panelHit.gameObject)); + } + + /// + /// Verifies the type name the bridge results are recognised by matches the uGUI type. + /// + [Test] + public void PanelRaycasterTypeFullName_MatchesUguiType() + { + Assert.That(UiRaycastHelper.PanelRaycasterTypeFullName, Is.EqualTo(typeof(PanelRaycaster).FullName)); + } + + private RaycastResult CreatePanelHit() + { + GameObject panelObject = CreateGameObject("Panel"); + return new RaycastResult { gameObject = panelObject, module = panelObject.AddComponent() }; + } + + private RaycastResult CreateGraphicHit() + { + GameObject canvasObject = CreateGameObject("Canvas"); + return new RaycastResult { gameObject = canvasObject, module = canvasObject.AddComponent() }; + } + + private GameObject CreateGameObject(string name) + { + GameObject gameObject = new GameObject("UiRaycastHelperPanelSkipTests_" + name); + _createdObjects.Add(gameObject); + return gameObject; + } + } +} diff --git a/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta new file mode 100644 index 000000000..20cc20769 --- /dev/null +++ b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2653f3ed2aaa646dc9186abf235ba686 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs new file mode 100644 index 000000000..c27f38a48 --- /dev/null +++ b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs @@ -0,0 +1,732 @@ +#nullable enable +using System.Collections; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.Runtime; +using io.github.hatayama.UnityCliLoop.ToolContracts; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEditor; +using UnityEngine; +using UnityEngine.EventSystems; +using UnityEngine.TestTools; +using UnityEngine.UI; +using UnityEngine.UIElements; + +using Image = UnityEngine.UI.Image; +using MouseButton = io.github.hatayama.UnityCliLoop.Runtime.MouseButton; +using UIToolkitButton = UnityEngine.UIElements.Button; + +namespace io.github.hatayama.UnityCliLoop.Tests.PlayMode +{ + /// + /// Verifies simulate-mouse-ui clicks and long presses reach UI Toolkit controls, choose between uGUI and UI + /// Toolkit where both are under the point as the EventSystem would, work without an EventSystem for UI + /// Toolkit, and report which UI system and path were hit. The UI Toolkit button sits at the centre of the + /// Game View, where the Canvas elements are anchored too. Each test decides whether an EventSystem exists. + /// + public sealed class SimulateMouseUiToolkitTests + { + private const string RuntimeThemePath = "Assets/UI Toolkit/UnityThemes/UnityDefaultRuntimeTheme.tss"; + private const string DocumentName = "SimulateMouseUiToolkitTests_Document"; + private const float ButtonWidth = 200f; + private const float ButtonHeight = 100f; + private const string NoEventSystemMessage = "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."; + + private readonly List _createdObjects = new List(); + private ExistingEventSystemDisableScope _eventSystemDisableScope = null!; + private GameObject _canvasObject = null!; + private SimulateMouseUiTool _tool = null!; + private SimulateMouseUiResponse _lastResponse = null!; + + [UnitySetUp] + public IEnumerator SetUp() + { + _eventSystemDisableScope = new ExistingEventSystemDisableScope(); + + _canvasObject = new GameObject("TestCanvas"); + _createdObjects.Add(_canvasObject); + Canvas canvas = _canvasObject.AddComponent(); + canvas.renderMode = RenderMode.ScreenSpaceOverlay; + _canvasObject.AddComponent(); + _canvasObject.AddComponent(); + + _tool = new SimulateMouseUiTool(); + yield return null; + } + + [UnityTearDown] + public IEnumerator TearDown() + { + MouseDragState.Clear(); + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.Destroy(createdObject); + } + } + + _createdObjects.Clear(); + yield return null; + + _eventSystemDisableScope.Restore(); + } + + /// + /// Verifies a uGUI click reports the UGUI system and the target's hierarchy path (#2). + /// + [UnityTest] + public IEnumerator Click_OnUgui_ReportsUguiSystemAndPath() + { + CreateEventSystem(); + ClickTracker tracker = CreateCanvasClickTarget("UguiTarget"); + yield return null; + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("UguiTarget", _lastResponse.HitGameObjectName); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + Assert.AreEqual("TestCanvas/UguiTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a click on a UI Toolkit button clicks it once, reports it, and leaves no pointer capture (#4). + /// + [UnityTest] + public IEnumerator Click_OnUIToolkitButtonWithEventSystem_ClicksIt() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("play", _lastResponse.HitGameObjectName); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + Assert.AreEqual(DocumentName + "#play", _lastResponse.HitPath); + Assert.AreEqual( + $"Clicked UI Toolkit element 'play' at ({GameViewCenter().x:F1}, {GameViewCenter().y:F1})", + _lastResponse.Message); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies a Canvas sorted above the panel takes the click (#5). + /// + [UnityTest] + public IEnumerator Click_WhereUguiCanvasSortsAbovePanel_ClicksUgui() + { + yield return ClickOverlap(canvasSortingOrder: 11, panelSortingOrder: 10f); + + Assert.IsTrue(_lastTracker.PointerClickCalled, "the Canvas element must be clicked"); + Assert.AreEqual(0, _lastCounter.Clicks, "the UI Toolkit button must not be clicked"); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + } + + /// + /// Verifies a Canvas and a panel with the same sorting order leave UI Toolkit in front (#6). + /// + [UnityTest] + public IEnumerator Click_WhereCanvasAndPanelShareSortingOrder_ClicksUIToolkit() + { + yield return ClickOverlap(canvasSortingOrder: 10, panelSortingOrder: 10f); + + Assert.IsFalse(_lastTracker.PointerClickCalled, "the Canvas element must not be clicked"); + Assert.AreEqual(1, _lastCounter.Clicks, "the UI Toolkit button must be clicked"); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a panel sorted above the Canvas takes the click (#7). + /// + [UnityTest] + public IEnumerator Click_WherePanelSortsAboveCanvas_ClicksUIToolkit() + { + yield return ClickOverlap(canvasSortingOrder: 9, panelSortingOrder: 10f); + + Assert.IsFalse(_lastTracker.PointerClickCalled, "the Canvas element must not be clicked"); + Assert.AreEqual(1, _lastCounter.Clicks, "the UI Toolkit button must be clicked"); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a UI Toolkit button is clicked when the scene has no EventSystem (#9). + /// + [UnityTest] + public IEnumerator Click_OnUIToolkitButtonWithoutEventSystem_ClicksIt() + { + Assert.IsNull(EventSystem.current, "Setup: no EventSystem must be active."); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a click with no EventSystem and no UI Toolkit element under the point fails and names both + /// UI systems (#10). + /// + [UnityTest] + public IEnumerator Click_WithoutEventSystemOnEmptyPoint_FailsNamingBothSystems() + { + Vector2 point = GameViewCenter(); + + yield return RunClick(point); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual( + $"No EventSystem found in the scene, and no UI Toolkit element is at ({point.x:F1}, {point.y:F1}). Ensure an EventSystem GameObject exists for uGUI.", + _lastResponse.Message); + } + + /// + /// Verifies a bypass click given a UI Toolkit path explains that such paths are not accepted (#15). + /// + [UnityTest] + public IEnumerator BypassClick_WithUIToolkitPath_ExplainsPathIsNotAccepted() + { + CreateEventSystem(); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + string path = DocumentName + "#play"; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = path + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual( + $"TargetPath '{path}' was not found. UI Toolkit element paths are not accepted here; click the element at its SimX/SimY without --bypass-raycast.", + _lastResponse.Message); + Assert.AreEqual(0, counter.Clicks); + } + + /// + /// Verifies a bypass click still needs an EventSystem and fails with the same message as before (#17). + /// + [UnityTest] + public IEnumerator BypassClick_WithoutEventSystem_FailsAsBefore() + { + CreateCanvasClickTarget("BypassTarget"); + yield return null; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = "TestCanvas/BypassTarget" + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual(NoEventSystemMessage, _lastResponse.Message); + } + + /// + /// Verifies option errors are reported before a missing EventSystem (#18). + /// + [UnityTest] + public IEnumerator Drag_WithNegativeDragSpeedAndNoEventSystem_ReportsDragSpeedError() + { + yield return RunTool(new JObject + { + ["action"] = MouseAction.Drag.ToString(), + ["dragSpeed"] = -1f + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual("DragSpeed must be non-negative, got: -1", _lastResponse.Message); + } + + /// + /// Verifies a drag with no EventSystem still fails with the existing message. + /// + [UnityTest] + public IEnumerator Drag_WithoutEventSystem_FailsAsBefore() + { + CreateDocumentWithCenteredButton(0f, out ClickCounter _); + yield return WaitForLayout(); + Vector2 point = GameViewCenter(); + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Drag.ToString(), + ["fromX"] = point.x, + ["fromY"] = point.y, + ["x"] = point.x + 10f, + ["y"] = point.y + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual(NoEventSystemMessage, _lastResponse.Message); + } + + /// + /// Verifies a right click reaches the UI Toolkit button as button 1 and does not click it (#20). + /// + [UnityTest] + public IEnumerator RightClick_OnUIToolkitButton_DoesNotClick() + { + CreateEventSystem(); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["x"] = GameViewCenter().x, + ["y"] = GameViewCenter().y, + ["button"] = MouseButton.Right.ToString() + }); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.LastPressedButton, "the press must arrive as the right button"); + Assert.AreEqual(0, counter.Clicks); + } + + /// + /// Verifies a click during a uGUI split drag is rejected as drag-active even after the EventSystem is + /// removed (#23). + /// + [UnityTest] + public IEnumerator Click_WhileSplitDragActiveWithoutEventSystem_IsRejectedAsDragActive() + { + GameObject eventSystemObject = CreateEventSystem(); + GameObject dragObject = CreateCanvasElement("DragTarget"); + dragObject.AddComponent(); + dragObject.AddComponent(); + yield return null; + Vector2 point = GameViewCenter(); + yield return RunTool(new JObject + { + ["action"] = MouseAction.DragStart.ToString(), + ["x"] = point.x, + ["y"] = point.y + }); + Assert.IsTrue(_lastResponse.Success, "Setup: DragStart must succeed. " + _lastResponse.Message); + eventSystemObject.SetActive(false); + yield return null; + + yield return RunClick(point); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual("Cannot Click while a split drag is active. Call DragEnd first.", _lastResponse.Message); + } + + /// + /// Verifies a handler-less uGUI image sorted above the panel stops the click from reaching UI Toolkit, + /// as it would for real input (#24). + /// + [UnityTest] + public IEnumerator Click_WhereHandlerlessUguiImageSortsAbovePanel_BlocksUIToolkit() + { + CreateEventSystem(); + _canvasObject.GetComponent().sortingOrder = 11; + CreateCanvasElement("Blocker").AddComponent(); + CreateDocumentWithCenteredButton(10f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(0, counter.Clicks, "the UI Toolkit button must not be clicked"); + Assert.IsNull(_lastResponse.HitGameObjectName); + Assert.IsNull(_lastResponse.UiSystem); + Assert.IsNull(_lastResponse.HitPath); + } + + /// + /// Verifies a click still succeeds without error when the UI Toolkit document is disabled after the + /// target is resolved and before the press is sent, and sends nothing: a document behind it on the same + /// panel would otherwise receive the press (#25). + /// + [UnityTest] + public IEnumerator Click_WhenUIToolkitDocumentDisabledBeforeDispatch_SkipsWithoutError() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + AddDocumentWithCenteredButton(document.panelSettings, "back", document.sortingOrder - 1f, out ClickCounter backCounter); + yield return WaitForLayout(); + Task task = StartTool(ClickParameters(GameViewCenter())); + + yield return null; + Assert.IsFalse(task.IsCompleted, "Setup: the click must still be animating when the document is disabled."); + document.enabled = false; + yield return WaitForTask(task); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual("play", _lastResponse.HitGameObjectName, "the front document must have been resolved"); + Assert.AreEqual(0, counter.PointerDowns); + Assert.AreEqual(0, backCounter.PointerDowns, "no press may reach the document behind"); + } + + /// + /// Verifies a Canvas image with no pointer handler reports no hit (#26). + /// + [UnityTest] + public IEnumerator Click_OnUguiImageWithoutHandler_ReportsNoHit() + { + CreateEventSystem(); + CreateCanvasElement("PlainImage").AddComponent(); + yield return null; + Vector2 point = GameViewCenter(); + + yield return RunClick(point); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual($"Clicked at ({point.x:F1}, {point.y:F1}) - no UI element hit", _lastResponse.Message); + Assert.IsNull(_lastResponse.HitGameObjectName); + Assert.IsNull(_lastResponse.UiSystem); + Assert.IsNull(_lastResponse.HitPath); + } + + /// + /// Verifies a click on a physics target outside any Canvas reports an empty UI system and its path (#27). + /// + [UnityTest] + public IEnumerator Click_OnPhysics2DTarget_ReportsEmptyUiSystem() + { + CreateEventSystem(); + GameObject physicsRoot = new GameObject("Physics2DRaycasterRoot"); + _createdObjects.Add(physicsRoot); + GameObject cameraObject = new GameObject("Physics2DCamera"); + cameraObject.transform.SetParent(physicsRoot.transform, false); + cameraObject.transform.position = new Vector3(0f, 0f, -10f); + Camera physicsCamera = cameraObject.AddComponent(); + physicsCamera.orthographic = true; + physicsCamera.orthographicSize = 5f; + cameraObject.AddComponent(); + GameObject physicsTarget = new GameObject("Physics2DTarget"); + physicsTarget.transform.SetParent(physicsRoot.transform, false); + Vector3 screenPoint = new Vector3(Screen.width / 2f, Screen.height / 2f, 10f); + Vector3 worldPoint = physicsCamera.ScreenToWorldPoint(screenPoint); + physicsTarget.transform.position = new Vector3(worldPoint.x, worldPoint.y, 0f); + physicsTarget.AddComponent().size = new Vector2(2f, 2f); + ClickTracker tracker = physicsTarget.AddComponent(); + yield return null; + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("Physics2DTarget", _lastResponse.HitGameObjectName); + Assert.AreEqual("", _lastResponse.UiSystem); + Assert.AreEqual("Physics2DRaycasterRoot/Physics2DTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a bypass click on a Canvas element reports the UGUI system and the target path (#28). + /// + [UnityTest] + public IEnumerator BypassClick_OnCanvasButton_ReportsUguiSystem() + { + CreateEventSystem(); + ClickTracker tracker = CreateCanvasClickTarget("BypassTarget"); + yield return null; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = "TestCanvas/BypassTarget" + }); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + Assert.AreEqual("TestCanvas/BypassTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a long press on a UI Toolkit button clicks it once on release, as UI Toolkit buttons do, and + /// leaves no pointer capture. + /// + [UnityTest] + public IEnumerator LongPress_OnUIToolkitButton_ClicksOnReleaseAndLeavesNoCapture() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunTool(LongPressParameters(GameViewCenter(), 0.2f)); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + Assert.AreEqual(DocumentName + "#play", _lastResponse.HitPath); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies a click whose UI Toolkit press handler throws still releases the pointer, and the exception + /// ends the command instead of being swallowed. + /// + [UnityTest] + public IEnumerator Click_WhenUIToolkitPressHandlerThrows_ReleasesAndPropagates() + { + yield return RunWithThrowingPressHandler(ClickParameters(GameViewCenter())); + } + + /// + /// Verifies a long press whose UI Toolkit press handler throws still releases the pointer, and the + /// exception ends the command instead of being swallowed. + /// + [UnityTest] + public IEnumerator LongPress_WhenUIToolkitPressHandlerThrows_ReleasesAndPropagates() + { + yield return RunWithThrowingPressHandler(LongPressParameters(GameViewCenter(), 0.2f)); + } + + /// + /// Verifies cancelling a long press while it holds still releases the UI Toolkit press, so no element is + /// left holding the pointer. + /// + [UnityTest] + public IEnumerator LongPress_OnUIToolkitButton_WhenCancelledDuringHold_ReleasesCapture() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + using CancellationTokenSource cancellation = new CancellationTokenSource(); + Task task = _tool.ExecuteAsync(LongPressParameters(GameViewCenter(), 5f), cancellation.Token); + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => counter.PointerDowns > 0 || Time.realtimeSinceStartup >= timeoutAt); + Assert.AreEqual(1, counter.PointerDowns, "Setup: the press must have arrived before cancelling."); + + cancellation.Cancel(); + timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => task.IsCompleted || Time.realtimeSinceStartup >= timeoutAt); + yield return null; + + Assert.IsTrue(task.IsCompleted, "the long press must end after cancelling"); + Assert.IsTrue(task.IsCanceled || task.Exception?.InnerException is System.OperationCanceledException, + $"the long press must end by cancellation, got {task.Status}: {task.Exception}"); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies a long press whose UI Toolkit document is disabled while it holds ends successfully without + /// error. + /// + [UnityTest] + public IEnumerator LongPress_WhenDocumentDisabledDuringHold_ReleasesWithoutError() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + Task task = StartTool(LongPressParameters(GameViewCenter(), 0.5f)); + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => counter.PointerDowns > 0 || Time.realtimeSinceStartup >= timeoutAt); + Assert.AreEqual(1, counter.PointerDowns, "Setup: the press must have arrived before disabling."); + + document.enabled = false; + yield return WaitForTask(task); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual("play", _lastResponse.HitGameObjectName); + } + + private IEnumerator RunWithThrowingPressHandler(JObject parameters) + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter _); + UIToolkitButton button = document.rootVisualElement.Q("play"); + int pointerUps = 0; + button.RegisterCallback( + _ => throw new System.InvalidOperationException("press handler failed"), + TrickleDown.TrickleDown); + button.RegisterCallback(_ => pointerUps++, TrickleDown.TrickleDown); + yield return WaitForLayout(); + + Task task = StartTool(parameters); + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => task.IsCompleted || Time.realtimeSinceStartup >= timeoutAt); + + Assert.IsTrue(task.IsCompleted, "Tool execution timed out."); + Assert.IsTrue(task.IsFaulted, $"the handler's exception must end the command, got {task.Status}"); + Assert.IsInstanceOf(task.Exception!.InnerException); + Assert.AreEqual(1, pointerUps, "the release must still be sent"); + } + + private ClickTracker _lastTracker = null!; + private ClickCounter _lastCounter = null!; + + private IEnumerator ClickOverlap(int canvasSortingOrder, float panelSortingOrder) + { + CreateEventSystem(); + _canvasObject.GetComponent().sortingOrder = canvasSortingOrder; + _lastTracker = CreateCanvasClickTarget("UguiTarget"); + CreateDocumentWithCenteredButton(panelSortingOrder, out _lastCounter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + } + + private GameObject CreateEventSystem() + { + GameObject eventSystemObject = new GameObject("TestEventSystem"); + _createdObjects.Add(eventSystemObject); + eventSystemObject.AddComponent(); + eventSystemObject.AddComponent(); + return eventSystemObject; + } + + private ClickTracker CreateCanvasClickTarget(string name) + { + GameObject element = CreateCanvasElement(name); + element.AddComponent(); + return element.AddComponent(); + } + + // Anchored at the Canvas centre, which is the Game View centre in PlayMode. + private GameObject CreateCanvasElement(string name) + { + GameObject element = new GameObject(name); + element.transform.SetParent(_canvasObject.transform, false); + RectTransform rect = element.AddComponent(); + rect.anchorMin = new Vector2(0.5f, 0.5f); + rect.anchorMax = new Vector2(0.5f, 0.5f); + rect.anchoredPosition = Vector2.zero; + rect.sizeDelta = new Vector2(ButtonWidth, ButtonHeight); + return element; + } + + // The object starts inactive so the document sees its panel settings when it first enables. + private UIDocument CreateDocumentWithCenteredButton(float sortingOrder, out ClickCounter counter) + { + PanelSettings settings = ScriptableObject.CreateInstance(); + settings.scaleMode = PanelScaleMode.ConstantPixelSize; + settings.scale = 1f; + settings.sortingOrder = sortingOrder; + settings.themeStyleSheet = AssetDatabase.LoadAssetAtPath(RuntimeThemePath); + Assert.IsNotNull(settings.themeStyleSheet, "the runtime theme must exist at " + RuntimeThemePath); + _createdObjects.Add(settings); + + return AddDocumentWithCenteredButton(settings, "play", 0f, out counter); + } + + // Documents sharing panel settings share one panel, ordered within it by the document's sorting order. + private UIDocument AddDocumentWithCenteredButton( + PanelSettings settings, + string buttonName, + float documentSortingOrder, + out ClickCounter counter) + { + GameObject documentObject = new GameObject(DocumentName); + documentObject.SetActive(false); + _createdObjects.Add(documentObject); + UIDocument document = documentObject.AddComponent(); + document.panelSettings = settings; + document.sortingOrder = documentSortingOrder; + documentObject.SetActive(true); + + UIToolkitButton button = new UIToolkitButton { name = buttonName }; + Vector2 center = GameViewCenter(); + button.style.position = Position.Absolute; + button.style.left = center.x - ButtonWidth / 2f; + button.style.top = center.y - ButtonHeight / 2f; + button.style.width = ButtonWidth; + button.style.height = ButtonHeight; + button.style.marginLeft = 0f; + button.style.marginTop = 0f; + counter = new ClickCounter(button); + document.rootVisualElement.Add(button); + return document; + } + + private static Vector2 GameViewCenter() + { + Vector2 size = Handles.GetMainGameViewSize(); + return new Vector2(Mathf.Round(size.x / 2f), Mathf.Round(size.y / 2f)); + } + + private static JObject ClickParameters(Vector2 point) + { + return new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["x"] = point.x, + ["y"] = point.y + }; + } + + private static JObject LongPressParameters(Vector2 point, float duration) + { + return new JObject + { + ["action"] = MouseAction.LongPress.ToString(), + ["x"] = point.x, + ["y"] = point.y, + ["duration"] = duration + }; + } + + private IEnumerator RunClick(Vector2 point) + { + yield return RunTool(ClickParameters(point)); + } + + private IEnumerator RunTool(JObject parameters) + { + yield return WaitForTask(StartTool(parameters)); + } + + private Task StartTool(JObject parameters) + { + return _tool.ExecuteAsync(parameters, CancellationToken.None); + } + + private IEnumerator WaitForTask(Task task) + { + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => task.IsCompleted || Time.realtimeSinceStartup >= timeoutAt); + Assert.IsTrue(task.IsCompleted, "Tool execution timed out."); + Assert.AreEqual(TaskStatus.RanToCompletion, task.Status, $"Tool execution should complete: {task.Exception}"); + _lastResponse = (SimulateMouseUiResponse)task.Result; + } + + // The root's size is set by the runtime panel update in the player loop, so frames must pass. + private static IEnumerator WaitForLayout() + { + yield return null; + yield return null; + } + + /// + /// Counts clicks and presses on a UI Toolkit button. + /// + private sealed class ClickCounter + { + internal ClickCounter(UIToolkitButton button) + { + button.clicked += () => Clicks++; + button.RegisterCallback( + evt => + { + PointerDowns++; + LastPressedButton = evt.button; + }, + TrickleDown.TrickleDown); + } + + internal int Clicks { get; private set; } + internal int PointerDowns { get; private set; } + internal int LastPressedButton { get; private set; } = -1; + } + } +} diff --git a/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta new file mode 100644 index 000000000..c6d8b4763 --- /dev/null +++ b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cbbacf9a9c5134f308de10793737999e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs new file mode 100644 index 000000000..f21a78079 --- /dev/null +++ b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs @@ -0,0 +1,397 @@ +using System.Collections; +using System.Collections.Generic; +using NUnit.Framework; +using UnityEditor; +using UnityEngine; +using UnityEngine.TestTools; +using UnityEngine.UIElements; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.Tests.PlayMode +{ + /// + /// Verifies which UI Toolkit control a Game View point resolves to on live runtime panels, and that presses, + /// moves and releases sent through the result reach the panel like real mouse input. Points are top-left + /// Game View pixels; panels use a scale of 1, so they equal panel coordinates. Every object is destroyed in + /// TearDown. + /// + public sealed class UIToolkitScreenElementsPointerTests + { + private const string Prefix = "UIToolkitScreenElementsPointerTests_"; + private const float Left = 100f; + private const float Top = 50f; + private const float Width = 200f; + private const float Height = 40f; + private const string RuntimeThemePath = "Assets/UI Toolkit/UnityThemes/UnityDefaultRuntimeTheme.tss"; + + private readonly List _createdObjects = new List(); + + [TearDown] + public void TearDown() + { + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.Destroy(createdObject); + } + } + + _createdObjects.Clear(); + } + + /// + /// Verifies that where two panels overlap, the control on the panel with the higher sorting order is + /// returned and the one behind it is not. + /// + [UnityTest] + public IEnumerator FindPointerTarget_WherePanelsOverlap_ReturnsFrontPanelControl() + { + UIDocument front = CreateDocument("Front", CreatePanelSettings(10f)); + AddPlaced(front.rootVisualElement, new Button { name = "front" }, Left, Top, Width, Height); + UIDocument back = CreateDocument("Back", CreatePanelSettings(0f)); + AddPlaced(back.rootVisualElement, new Button { name = "back" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("front")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Front#front")); + Assert.That(target.SortingOrder, Is.EqualTo(10f)); + } + + /// + /// Verifies a point on a button reports the button's name and its annotation-style path. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnButton_ReportsNameAndPath() + { + UIDocument document = CreateDocument("Doc", CreatePanelSettings(0f)); + AddPlaced(document.rootVisualElement, new Button { name = "play" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("play")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Doc#play")); + } + + /// + /// Verifies a point on a part inside a toggle still reports the toggle. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnPartInsideToggle_ReportsToggle() + { + UIDocument document = CreateDocument("Toggle", CreatePanelSettings(0f)); + Toggle toggle = new Toggle { name = "sound" }; + AddPlaced(document.rootVisualElement, toggle, Left, Top, Width, Height); + yield return WaitForLayout(); + VisualElement checkmark = toggle.Q(className: Toggle.checkmarkUssClassName); + Vector2 point = checkmark.worldBound.center; + Assert.That(document.rootVisualElement.panel.Pick(point), Is.Not.SameAs(toggle), "the point must hit a part inside the toggle"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("sound")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Toggle#sound")); + } + + /// + /// Verifies points on the dragger and both arrow buttons of a scroll view's scroller all report the + /// scroller with the same path screenshot annotations list for it, not the slider or buttons inside. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnScrollerParts_ReportsScrollerWithAnnotationPath() + { + UIDocument document = CreateDocument("Scroll", CreatePanelSettings(0f)); + ScrollView scrollView = CreateScrollView(document.rootVisualElement); + yield return WaitForLayout(); + Scroller scroller = scrollView.verticalScroller; + UIElementInfo annotated = FindAnnotated("Scroll", "Scrollbar"); + VisualElement[] parts = { scroller.slider.Q("unity-dragger"), scroller.lowButton, scroller.highButton }; + + foreach (VisualElement part in parts) + { + Vector2 point = part.worldBound.center; + Assert.That(document.rootVisualElement.panel.Pick(point), Is.Not.SameAs(scroller), "the point must hit a part inside the scroller"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null, part.name); + Assert.That(target.Path, Is.EqualTo(annotated.Path), part.name); + Assert.That(target.Name, Is.EqualTo(annotated.Name), part.name); + } + } + + /// + /// Verifies a point on the input field of a slider that shows one reports the slider. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnSliderInputField_ReportsSlider() + { + UIDocument document = CreateDocument("Slider", CreatePanelSettings(0f)); + Slider slider = new Slider(0f, 1f) { name = "volume", showInputField = true }; + AddPlaced(document.rootVisualElement, slider, Left, Top, 300f, 30f); + yield return WaitForLayout(); + TextField inputField = slider.Q(); + Vector2 point = inputField.worldBound.center; + VisualElement picked = document.rootVisualElement.panel.Pick(point); + Assert.That(picked == inputField || inputField.Contains(picked), Is.True, "the point must hit the input field"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Path, Is.EqualTo(Prefix + "Slider#volume")); + } + + /// + /// Verifies a point with no element under it returns null. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnEmptyPoint_ReturnsNull() + { + UIDocument document = CreateDocument("Empty", CreatePanelSettings(0f)); + AddPlaced(document.rootVisualElement, new Button { name = "play" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + Assert.That(UIToolkitScreenElements.FindPointerTarget(new Vector2(Left + Width + 50f, Top + Height + 50f)), Is.Null); + } + + /// + /// Verifies a front panel that renders into a texture still takes the point, as Unity still routes + /// screen input to it. + /// + [UnityTest] + public IEnumerator FindPointerTarget_FrontPanelWithTargetTexture_ReturnsItsControl() + { + UIDocument back = CreateDocument("TextureBack", CreatePanelSettings(0f)); + AddPlaced(back.rootVisualElement, new Button { name = "target" }, Left, Top, Width, Height); + PanelSettings frontSettings = CreatePanelSettings(10f); + frontSettings.targetTexture = CreateScreenSizedTexture(); + UIDocument front = CreateDocument("TextureFront", frontSettings); + AddPlaced(front.rootVisualElement, new Button { name = "cover" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("cover")); + } + + /// + /// Verifies a left press and release click a button once and leave no element holding the pointer. + /// + [UnityTest] + public IEnumerator PressDownThenRelease_OnButton_ClicksOnceAndLeavesNoCapture() + { + UIDocument document = CreateDocument("Click", CreatePanelSettings(0f)); + Button button = new Button { name = "play" }; + int clicks = 0; + button.clicked += () => clicks++; + AddPlaced(document.rootVisualElement, button, Left, Top, Width, Height); + yield return WaitForLayout(); + Vector2 point = Center(Left, Top, Width, Height); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + Assert.That(target, Is.Not.Null); + + target.PressDown(point, 0); + target.Release(point, 0); + + Assert.That(clicks, Is.EqualTo(1)); + Assert.That(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId), Is.Null); + } + + /// + /// Verifies that once the target's document is disabled the target reports itself unavailable, and + /// presses, moves and releases throw nothing and send nothing to the panel, which another document + /// still shows. + /// + [UnityTest] + public IEnumerator PointerCalls_AfterDocumentDisabled_SendNothing() + { + PanelSettings sharedSettings = CreatePanelSettings(0f); + UIDocument disabled = CreateDocument("Disabled", sharedSettings); + AddPlaced(disabled.rootVisualElement, new Button { name = "gone" }, Left, Top, Width, Height); + UIDocument remaining = CreateDocument("Remaining", sharedSettings); + Button witness = new Button { name = "witness" }; + AddPlaced(remaining.rootVisualElement, witness, Left, Top + 100f, Width, Height); + int received = 0; + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + yield return WaitForLayout(); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + Assert.That(target, Is.Not.Null); + Vector2 witnessPoint = Center(Left, Top + 100f, Width, Height); + + disabled.enabled = false; + yield return null; + + Assert.That(target.IsAvailable, Is.False); + Assert.DoesNotThrow(() => target.PressDown(witnessPoint, 0)); + Assert.DoesNotThrow(() => target.MoveTo(witnessPoint)); + Assert.DoesNotThrow(() => target.Release(witnessPoint, 0)); + Assert.That(received, Is.EqualTo(0)); + + // The callbacks do fire when a target that is still available sends to the same point. + UIToolkitPointerTarget witnessTarget = UIToolkitScreenElements.FindPointerTarget(witnessPoint); + witnessTarget.PressDown(witnessPoint, 0); + witnessTarget.Release(witnessPoint, 0); + Assert.That(received, Is.GreaterThan(0)); + } + + /// + /// Verifies a right press and release reach the button as button 1 and do not click it. + /// + [UnityTest] + public IEnumerator PressDownThenRelease_WithRightButton_DoesNotClick() + { + UIDocument document = CreateDocument("Right", CreatePanelSettings(0f)); + Button button = new Button { name = "play" }; + int clicks = 0; + int pressedButton = -1; + button.clicked += () => clicks++; + button.RegisterCallback(evt => pressedButton = evt.button, TrickleDown.TrickleDown); + AddPlaced(document.rootVisualElement, button, Left, Top, Width, Height); + yield return WaitForLayout(); + Vector2 point = Center(Left, Top, Width, Height); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + Assert.That(target, Is.Not.Null); + + target.PressDown(point, 1); + target.Release(point, 1); + + Assert.That(pressedButton, Is.EqualTo(1)); + Assert.That(clicks, Is.EqualTo(0)); + } + + /// + /// Verifies an item of an open dropdown menu, which belongs to no document, resolves with a null path + /// and pressing it changes the dropdown's value. + /// + [UnityTest] + public IEnumerator PressOnOpenDropdownMenuItem_ChangesValueAndHasNoPath() + { + UIDocument document = CreateDocument("Dropdown", CreatePanelSettings(0f)); + DropdownField dropdown = new DropdownField(new List { "low", "high" }, 0) { name = "quality" }; + AddPlaced(document.rootVisualElement, dropdown, Left, Top, Width, 30f); + yield return WaitForLayout(); + Vector2 dropdownPoint = Center(Left, Top, Width, 30f); + UIToolkitPointerTarget opener = UIToolkitScreenElements.FindPointerTarget(dropdownPoint); + Assert.That(opener, Is.Not.Null); + Assert.That(opener.Path, Is.EqualTo(Prefix + "Dropdown#quality")); + opener.PressDown(dropdownPoint, 0); + opener.Release(dropdownPoint, 0); + yield return WaitForLayout(); + List items = document.rootVisualElement.panel.visualTree + .Query(className: GenericDropdownMenu.itemUssClassName).ToList(); + Assert.That(items.Count, Is.EqualTo(2), "the dropdown menu must be open"); + Vector2 itemPoint = items[1].worldBound.center; + + UIToolkitPointerTarget item = UIToolkitScreenElements.FindPointerTarget(itemPoint); + + Assert.That(item, Is.Not.Null); + Assert.That(item.Path, Is.Null); + item.PressDown(itemPoint, 0); + item.Release(itemPoint, 0); + Assert.That(dropdown.value, Is.EqualTo("high")); + } + + private ScrollView CreateScrollView(VisualElement root) + { + ScrollView scrollView = new ScrollView { name = "list" }; + scrollView.verticalScrollerVisibility = ScrollerVisibility.AlwaysVisible; + scrollView.horizontalScrollerVisibility = ScrollerVisibility.Hidden; + AddPlaced(root, scrollView, Left, Top, Width, 100f); + Scroller scroller = scrollView.verticalScroller; + SetSize(scroller, 20f, 100f); + SetSize(scroller.lowButton, 20f, 20f); + SetSize(scroller.highButton, 20f, 20f); + SetSize(scroller.slider, 20f, 60f); + // The scroll view disables a scroller whose content fits, so the viewport is sized and given + // content taller than itself that does not shrink to fit. + SetSize(scrollView.contentViewport, 180f, 100f); + VisualElement content = new VisualElement { name = "content" }; + SetSize(content, 180f, 500f); + content.style.flexShrink = 0f; + scrollView.Add(content); + return scrollView; + } + + private static UIElementInfo FindAnnotated(string documentName, string type) + { + List elements = new List(); + UIToolkitElementCollector.Collect(elements, Handles.GetMainGameViewSize().y); + List matches = elements.FindAll( + element => element.Type == type && element.Path.StartsWith(Prefix + documentName + "#")); + Assert.That(matches.Count, Is.EqualTo(1), "the annotation must list exactly one " + type); + return matches[0]; + } + + private PanelSettings CreatePanelSettings(float sortingOrder) + { + PanelSettings settings = ScriptableObject.CreateInstance(); + settings.scaleMode = PanelScaleMode.ConstantPixelSize; + settings.scale = 1f; + settings.sortingOrder = sortingOrder; + + // Without a theme, the parts inside controls get no size and the dropdown menu finds no root + // container to open in, so the points these tests aim at would not exist. + settings.themeStyleSheet = AssetDatabase.LoadAssetAtPath(RuntimeThemePath); + Assert.That(settings.themeStyleSheet, Is.Not.Null, "the runtime theme must exist at " + RuntimeThemePath); + _createdObjects.Add(settings); + return settings; + } + + private RenderTexture CreateScreenSizedTexture() + { + RenderTexture texture = new RenderTexture(Screen.width, Screen.height, 0); + _createdObjects.Add(texture); + return texture; + } + + // The object starts inactive so the document sees its panel settings when it first enables. + private UIDocument CreateDocument(string name, PanelSettings settings) + { + GameObject documentObject = new GameObject(Prefix + name); + documentObject.SetActive(false); + _createdObjects.Add(documentObject); + UIDocument document = documentObject.AddComponent(); + document.panelSettings = settings; + documentObject.SetActive(true); + return document; + } + + private static void AddPlaced(VisualElement parent, VisualElement element, float left, float top, float width, float height) + { + element.style.position = Position.Absolute; + element.style.left = left; + element.style.top = top; + SetSize(element, width, height); + parent.Add(element); + } + + private static void SetSize(VisualElement element, float width, float height) + { + element.style.width = width; + element.style.height = height; + } + + private static Vector2 Center(float left, float top, float width, float height) + { + return new Vector2(left + width / 2f, top + height / 2f); + } + + // The root's size is set by the runtime panel update in the player loop, so frames must pass. + private static IEnumerator WaitForLayout() + { + yield return null; + yield return null; + } + } +} diff --git a/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta new file mode 100644 index 000000000..12888fdd0 --- /dev/null +++ b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f91b3eaa63d5945eea43a7aebdd41262 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Documentation~/tools.md b/Packages/src/Documentation~/tools.md index efef7ade9..172653a7a 100644 --- a/Packages/src/Documentation~/tools.md +++ b/Packages/src/Documentation~/tools.md @@ -158,7 +158,7 @@ When enabled, dynamic code execution runs with full Unity Editor process permiss ### 14. simulate-mouse-ui - Simulate Mouse Input on PlayMode UI Simulate mouse click, long-press, and drag on PlayMode UI elements. Uses EventSystem and ExecuteEvents to dispatch pointer events directly — works independently of both old and new Input System. For game logic that reads Input System (e.g. `Mouse.current.leftButton.wasPressedThisFrame`), use `simulate-mouse-input` instead. -Supports 6 actions: Click, LongPress, Drag (one-shot), DragStart/DragMove/DragEnd (split drag). +Supports 6 actions: Click, LongPress, Drag (one-shot), DragStart/DragMove/DragEnd (split drag). Click and LongPress also reach UI Toolkit controls at their annotated coordinates, without an EventSystem. ```text → screenshot (CaptureMode: rendering, AnnotateElements: true) @@ -221,4 +221,4 @@ sh scripts/run-posix-e2e.sh --project-path /path/to/unity-project powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\run-windows-e2e.ps1 ``` -`run-posix-e2e.sh` uses the built native CLI binary by default, passes an explicit `--project-path` to every `uloop` invocation, and runs CLI recovery/readiness, simulate-mouse UI coverage, and UI Toolkit screenshot annotation coverage in one sequence (skip the last with `--skip-uitoolkit-annotation`). To verify `replay-input` against a JSON created in the Recordings window, run `verify-replay-via-cli.sh` separately. +`run-posix-e2e.sh` uses the built native CLI binary by default, passes an explicit `--project-path` to every `uloop` invocation, and runs CLI recovery/readiness, simulate-mouse UI coverage, and UI Toolkit screenshot annotation and click coverage in one sequence (skip the last with `--skip-uitoolkit-annotation`). To verify `replay-input` against a JSON created in the Recordings window, run `verify-replay-via-cli.sh` separately. diff --git a/Packages/src/Documentation~/tools_ja.md b/Packages/src/Documentation~/tools_ja.md index a336447c5..411720f30 100644 --- a/Packages/src/Documentation~/tools_ja.md +++ b/Packages/src/Documentation~/tools_ja.md @@ -158,7 +158,7 @@ Unity Editor内で動的にC#コードを実行します。 ### 14. simulate-mouse-ui - PlayMode UI要素のマウス操作シミュレーション PlayMode中のUI要素に対してマウスクリック・長押し・ドラッグをシミュレーションします。EventSystemとExecuteEventsを使ってポインタイベントを直接ディスパッチするため、旧Input System・新Input Systemの両方に依存せず動作します。ゲームロジックがInput Systemを直接読み取る場合（例：`Mouse.current.leftButton.wasPressedThisFrame`）は、`simulate-mouse-input` を使用してください。 -6つのアクションに対応: Click、LongPress、Drag（ワンショット）、DragStart/DragMove/DragEnd（分割ドラッグ） +6つのアクションに対応: Click、LongPress、Drag（ワンショット）、DragStart/DragMove/DragEnd（分割ドラッグ）。Click と LongPress は、注釈の座標で UI Toolkit のコントロールにも届きます（EventSystem は不要）。 ```text → screenshot (CaptureMode: rendering, AnnotateElements: true) @@ -221,4 +221,4 @@ sh scripts/run-posix-e2e.sh --project-path /path/to/unity-project powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\run-windows-e2e.ps1 ``` -`run-posix-e2e.sh` は、デフォルトでチェックイン済みのネイティブCLIバイナリを使い、すべての `uloop` 呼び出しに明示的な `--project-path` を渡します。CLI recovery/readiness、simulate-mouse UI、UI Toolkit のスクリーンショット注釈を1つの流れで検証します（最後のものは `--skip-uitoolkit-annotation` で省けます）。Recordingsウィンドウで作成したJSONの replay-input 検証は `verify-replay-via-cli.sh` を個別に実行します。 +`run-posix-e2e.sh` は、デフォルトでチェックイン済みのネイティブCLIバイナリを使い、すべての `uloop` 呼び出しに明示的な `--project-path` を渡します。CLI recovery/readiness、simulate-mouse UI、UI Toolkit のスクリーンショット注釈とクリックを1つの流れで検証します（最後のものは `--skip-uitoolkit-annotation` で省けます）。Recordingsウィンドウで作成したJSONの replay-input 検証は `verify-replay-via-cli.sh` を個別に実行します。 diff --git a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs index 356af9dc1..b6774147d 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs @@ -12,12 +12,58 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// internal static class UiRaycastHelper { + // Results from uGUI's bridge to UI Toolkit panels are recognised by type name because the type is + // compiled only when the UI Toolkit package is present. They are skipped where a caller resolves UI + // Toolkit itself: the bridge flips Y with Screen.height, which outside the player loop is the size of + // the last Editor view drawn, so its hits land at the wrong place on the panel. + internal const string PanelRaycasterTypeFullName = "UnityEngine.UIElements.PanelRaycaster"; + public static RaycastResult? RaycastUI(Vector2 screenPosition, EventSystem eventSystem) { RaycastContext context = new(eventSystem); return context.Raycast(screenPosition); } + internal static RaycastResult? RaycastUISkippingUIToolkitPanels(Vector2 screenPosition, EventSystem eventSystem) + { + RaycastContext context = new(eventSystem); + return context.RaycastSkippingUIToolkitPanels(screenPosition); + } + + // Picks the EventSystem's first hit, unless the Canvas-space hit should win over it; with no + // EventSystem hit, the Canvas-space hit. + internal static RaycastResult? ChooseHit(List eventSystemResults, RaycastResult? canvasSpaceHit) + { + if (eventSystemResults.Count > 0) + { + RaycastResult firstHit = eventSystemResults[0]; + + if (canvasSpaceHit != null && ShouldPreferCanvasSpaceHit(canvasSpaceHit.Value, firstHit)) + { + return canvasSpaceHit; + } + + return firstHit; + } + + // EventSystem clips at Screen.width/height, which can be smaller than the + // Canvas layout space (Game view target resolution). Fall back to manual hit testing. + return canvasSpaceHit; + } + + internal static RaycastResult? ChooseHitSkippingUIToolkitPanels( + List eventSystemResults, + RaycastResult? canvasSpaceHit) + { + List withoutPanels = eventSystemResults.FindAll(result => !IsPanelRaycasterHit(result)); + return ChooseHit(withoutPanels, canvasSpaceHit); + } + + private static bool IsPanelRaycasterHit(RaycastResult result) + { + return result.module != null && result.module.GetType().FullName == PanelRaycasterTypeFullName; + } + private static List CollectCanvasRaycastSources() { #if UNITY_6000_4_OR_NEWER @@ -106,27 +152,24 @@ internal RaycastContext(EventSystem eventSystem) } public RaycastResult? Raycast(Vector2 screenPosition) + { + RaycastResult? canvasSpaceHit = RaycastBothWays(screenPosition); + return ChooseHit(_eventSystemResults, canvasSpaceHit); + } + + internal RaycastResult? RaycastSkippingUIToolkitPanels(Vector2 screenPosition) + { + RaycastResult? canvasSpaceHit = RaycastBothWays(screenPosition); + return ChooseHitSkippingUIToolkitPanels(_eventSystemResults, canvasSpaceHit); + } + + // Fills the EventSystem results and returns the Canvas-space hit for the same point. + private RaycastResult? RaycastBothWays(Vector2 screenPosition) { _eventSystemResults.Clear(); _pointerData.position = screenPosition; _eventSystem.RaycastAll(_pointerData, _eventSystemResults); - RaycastResult? canvasSpaceHit = RaycastCanvasSpaceFromSources(screenPosition, _canvasRaycastSources); - - if (_eventSystemResults.Count > 0) - { - RaycastResult firstHit = _eventSystemResults[0]; - - if (canvasSpaceHit != null && ShouldPreferCanvasSpaceHit(canvasSpaceHit.Value, firstHit)) - { - return canvasSpaceHit; - } - - return firstHit; - } - - // EventSystem clips at Screen.width/height, which can be smaller than the - // Canvas layout space (Game view target resolution). Fall back to manual hit testing. - return canvasSpaceHit; + return RaycastCanvasSpaceFromSources(screenPosition, _canvasRaycastSources); } } diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs new file mode 100644 index 000000000..47e9bee2b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs @@ -0,0 +1,55 @@ +#nullable enable +using System; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Sends mouse pointer events to a runtime panel the way Unity turns real mouse input into them. + /// + internal static class UIToolkitPointerEvents + { + // No target is set, so the panel hands the event to the element holding the pointer or, failing that, + // the element under it, as it does for real input. GetPooled also records the press or release in the + // pointer device state, which fills pressedButtons for later events. + internal static void Send(IPanel panel, EventType type, Vector2 panelPoint, Vector2 delta, int button) + { + Event systemEvent = new Event + { + type = type, + mousePosition = panelPoint, + delta = delta, + button = button, + clickCount = type == EventType.MouseDown || type == EventType.MouseUp ? 1 : 0 + }; + + switch (type) + { + case EventType.MouseDown: + using (PointerDownEvent pointerDown = PointerDownEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerDown); + } + + return; + case EventType.MouseMove: + using (PointerMoveEvent pointerMove = PointerMoveEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerMove); + } + + return; + case EventType.MouseUp: + using (PointerUpEvent pointerUp = PointerUpEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerUp); + } + + return; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, "Only mouse down, move and up are sent."); + } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta new file mode 100644 index 000000000..a36928167 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2dfc472d65b414f7ca31681699b8e0f1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs new file mode 100644 index 000000000..0a4fb7fda --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs @@ -0,0 +1,108 @@ +#nullable enable +using System; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The UI Toolkit control under a Game View point, and the panel that takes pointer input there. Presses, + /// moves and releases are sent to that panel the way real input reaches it, so they land on whichever + /// element is under the pointer or holds the pointer at that moment. + /// + public sealed class UIToolkitPointerTarget + { + private const int MaxMouseButton = 2; + + private readonly IPanel _panel; + private readonly UIDocument _document; + private Vector2 _lastPanelPoint; + + internal UIToolkitPointerTarget(IPanel panel, UIDocument document, string name, string? path, float sortingOrder) + { + if (string.IsNullOrEmpty(name)) + { + // Callers treat a null name as "nothing was hit", so a hit must always carry one. + throw new ArgumentException("A UI Toolkit pointer target needs a display name.", nameof(name)); + } + + _panel = panel ?? throw new ArgumentNullException(nameof(panel)); + _document = document != null ? document : throw new ArgumentNullException(nameof(document)); + Name = name; + Path = path; + SortingOrder = sortingOrder; + } + + /// The control's name, or its type name when it has none. Never null or empty. + public string Name { get; } + + /// + /// The same path screenshot annotations report for the control, or null when the element belongs to no + /// document, as with the menu of an open dropdown. + /// + public string? Path { get; } + + /// The sorting order of the panel the control is on, as set in its panel settings. + public float SortingOrder { get; } + + /// False once the document that owns the panel has been disabled or destroyed. + public bool IsAvailable => + _document != null && _document.isActiveAndEnabled && _document.rootVisualElement?.panel == _panel; + + /// Presses a mouse button (0 left, 1 right, 2 middle) at a Game View point. + /// Does nothing once the target is no longer available. + public void PressDown(Vector2 gameViewTopLeft, int button) + { + ValidateButton(button); + if (!IsAvailable) + { + // The document can go away between resolving the target and pressing, a frame or more later; + // uGUI likewise sends nothing to a destroyed or inactive target. + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + UIToolkitPointerEvents.Send(_panel, EventType.MouseDown, panelPoint, Vector2.zero, button); + _lastPanelPoint = panelPoint; + } + + /// Moves the pointer to a Game View point, carrying any buttons still held. + /// Does nothing once the target is no longer available. + public void MoveTo(Vector2 gameViewTopLeft) + { + if (!IsAvailable) + { + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + + // The button is overwritten with "none" for moves; held buttons travel in pressedButtons. + UIToolkitPointerEvents.Send(_panel, EventType.MouseMove, panelPoint, panelPoint - _lastPanelPoint, 0); + _lastPanelPoint = panelPoint; + } + + /// Releases a mouse button (0 left, 1 right, 2 middle) at a Game View point. + /// Does nothing once the target is no longer available, so cleanup code can always call it. + public void Release(Vector2 gameViewTopLeft, int button) + { + ValidateButton(button); + if (!IsAvailable) + { + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + UIToolkitPointerEvents.Send(_panel, EventType.MouseUp, panelPoint, panelPoint - _lastPanelPoint, button); + _lastPanelPoint = panelPoint; + } + + private static void ValidateButton(int button) + { + if (button < 0 || button > MaxMouseButton) + { + throw new ArgumentOutOfRangeException(nameof(button), button, "Mouse button must be 0, 1 or 2."); + } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta new file mode 100644 index 000000000..12ad95392 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 71c2930e001654e8e8c062c127c171e3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs new file mode 100644 index 000000000..07a3c911c --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs @@ -0,0 +1,82 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides which panel takes pointer input at a Game View point and which control to report for it. + /// + internal static class UIToolkitPointerTargetFinder + { + // Expects the panels' layouts to be settled already, because picking reads them. The first panel, front + // to back, that picks anything takes the input, as Unity routes it. Panels with the same sorting order + // keep the order they were gathered in, the same known limit as the annotation's cover check. + internal static UIToolkitPointerTarget? Find(List panels, Vector2 gameViewTopLeft) + { + foreach (UIToolkitScreenPanels.PanelEntry entry in panels.OrderByDescending(panel => panel.SortingOrder)) + { + VisualElement? picked = UIToolkitScreenPanels.PickInPanel(entry.Panel, gameViewTopLeft); + if (picked == null) + { + continue; + } + + return CreateTarget(entry, picked); + } + + return null; + } + + private static UIToolkitPointerTarget CreateTarget(UIToolkitScreenPanels.PanelEntry entry, VisualElement picked) + { + UIDocument? document = FindOwningDocument(entry, picked); + if (document == null) + { + // Elements outside every document, such as the menu of an open dropdown, have no annotation path. + // The panel's first document stands in to tell when the panel goes away. + return new UIToolkitPointerTarget( + entry.Panel, entry.Documents[0], UIToolkitElementPath.GetDisplayName(picked), null, entry.SortingOrder); + } + + VisualElement reported = FindOutermostControl(document.rootVisualElement, picked) ?? picked; + return new UIToolkitPointerTarget( + entry.Panel, + document, + UIToolkitElementPath.GetDisplayName(reported), + UIToolkitElementPath.Build(document, reported), + entry.SortingOrder); + } + + private static UIDocument? FindOwningDocument(UIToolkitScreenPanels.PanelEntry entry, VisualElement picked) + { + for (VisualElement? current = picked; current != null; current = current.hierarchy.parent) + { + UIDocument? document = entry.Documents.Find(candidate => candidate.rootVisualElement == current); + if (document != null) + { + return document; + } + } + + return null; + } + + // The outermost, not the nearest, control: annotations stop at the first control on the way down from + // the root and never list the parts inside it. A Scroller holds a Slider and two RepeatButtons, and a + // Slider with an input field holds a TextField; reporting those parts would not match the annotation. + private static VisualElement? FindOutermostControl(VisualElement root, VisualElement picked) + { + List chain = new List(); + for (VisualElement? current = picked; current != null && current != root; current = current.hierarchy.parent) + { + chain.Add(current); + } + + chain.Reverse(); + return chain.Find(element => UIToolkitElementClassifier.Classify(element) != null); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta new file mode 100644 index 000000000..60e91f6c6 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 644b0b96772bd4ed0bfd525bc25d378f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs index c8d69dfe2..88c49d81d 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs @@ -8,7 +8,7 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// public sealed class UIToolkitScreenControl { - public UIToolkitScreenControl( + internal UIToolkitScreenControl( string name, string path, string type, diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs index f3351c571..25f57968a 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs @@ -32,5 +32,17 @@ public static IReadOnlyList CollectControls( UIToolkitScreenPanels.SettleLayouts(panels); return UIToolkitControlCollector.Collect(panels, probePointsForTopLeftBounds); } + + /// + /// Returns the control that pointer input at a Game View point reaches, or null when no panel takes + /// input there. Panels are tried front to back, as Unity routes input. The control reported is the + /// outermost control around the element under the point, the one screenshot annotations list. + /// + public static UIToolkitPointerTarget? FindPointerTarget(Vector2 gameViewTopLeft) + { + List panels = UIToolkitScreenPanels.Gather(); + UIToolkitScreenPanels.SettleLayouts(panels); + return UIToolkitPointerTargetFinder.Find(panels, gameViewTopLeft); + } } } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md index 5e1338e01..69466c096 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/Skill/references/annotated-elements.md @@ -10,7 +10,7 @@ Device Simulator is supported for this flow: prefer `--capture-mode rendering` ( - `Label`: Index label in JSON (`A` = first in that sort, `B` = next, ...). Treat it as a stable handle, not as proof of what is actually in front: when overlapping elements cross parents, pick the target by `Path` and confirm the result with `simulate-mouse-ui`'s `HitGameObjectName` — the handler that received the events, resolved up the hierarchy from the raycast hit (or from `--target-path` in bypass mode), so it can name an ancestor of the element you clicked. Screenshot labels also include the interaction hint, such as `A / CLICK` or `B / DRAG`. - `Name`: Element name -- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; see [UI Toolkit Elements](#ui-toolkit-elements). +- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers. UI Toolkit entries use a different form that `--target-path` does not accept; click them at `SimX`/`SimY` instead. See [UI Toolkit Elements](#ui-toolkit-elements). - `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`, `PhysicsCollider`) - `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`) or `Raycast` for clustered physics collider entries. Use this to choose between `simulate-mouse-ui --action Click`, drag actions, or `simulate-mouse-input` (including `--dry-run`). - `UiSystem`: UI system the entry came from: `UGUI`, `UIToolkit`, or empty for `PhysicsCollider` entries. @@ -53,9 +53,9 @@ Limits: Clicking: -- `simulate-mouse-input --action Click --x --y ` reaches UI Toolkit controls when the scene has an EventSystem with an Input System UI input module. -- Without an EventSystem the click does not reach UI Toolkit controls. -- `simulate-mouse-ui` does not reach UI Toolkit controls, and its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-ui --action Click --x --y ` (or `LongPress`) reaches UI Toolkit controls, with or without an EventSystem. Its `--target-path` does not accept UI Toolkit `Path` values. +- `simulate-mouse-input --action Click --x --y ` also reaches them when the scene has an EventSystem with an Input System UI input module. +- Without an EventSystem the `simulate-mouse-input` click does not reach UI Toolkit controls. ### PhysicsCollider Entries Per Closed Region diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md index d7c92b3d4..41723aa25 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/SKILL.md @@ -67,7 +67,7 @@ All rows below assume the New Input System is installed. | Scenario | Tool | |----------|------| -| Click a Unity UI Button (IPointerClickHandler) | `simulate-mouse-ui` | +| Click a uGUI or UI Toolkit button | `simulate-mouse-ui` | | Runtime logic reads `Mouse.current.leftButton` | `simulate-mouse-input` | | Runtime logic reads right-click | `simulate-mouse-input --button Right` | | Drag a UI slider | `simulate-mouse-ui --action Drag` | @@ -110,7 +110,7 @@ uloop simulate-mouse-input --action SmoothDelta --delta-x 300 --delta-y 0 --dura ## Coordinate System `--x`/`--y` use top-left Game View coordinates: pass `AnnotatedElements[].SimX/SimY` -directly (UI Toolkit entries need an EventSystem in the scene), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y +directly (UI Toolkit entries need an EventSystem with an Input System UI module here; `simulate-mouse-ui` reaches them without one), convert raw screenshot pixels with `ScreenshotToInputFormula`, and never flip Y in the caller (the tool converts internally). Conversion formula, Device Simulator notes, and how the injected position reads back inside Unity are in `references/output-and-coordinates.md`. diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/references/output-and-coordinates.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/references/output-and-coordinates.md index a5998020b..979f721cb 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/references/output-and-coordinates.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseInput/Skill/references/output-and-coordinates.md @@ -30,7 +30,7 @@ Verify visual outcome with a follow-up screenshot. - `--x` / `--y` use **top-left Game View coordinates**. - Raw image pixels from `uloop screenshot --capture-mode rendering` must be converted with `ScreenshotToInputFormula`. -- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. +- `AnnotatedElements[].SimX/SimY` can be passed directly to this tool. For UI Toolkit entries (`UiSystem` `UIToolkit`), a `Click` reaches the control only when the scene has an EventSystem with an Input System UI input module; without one, nothing receives the click. `simulate-mouse-ui` reaches them without an EventSystem. - Do not flip Y in the caller. The tool converts internally for Unity Input System: ```text diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs new file mode 100644 index 000000000..271329562 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The thing a click or long press is delivered to, whichever UI system it belongs to. + /// + internal interface IMouseUiPressTarget + { + // Captured when the target is created, so it stays valid after the target goes away. + MouseUiHitReport Report { get; } + + void Click(); + + void PressDown(); + + // Safe to call from cleanup even if nothing was pressed or the target has gone away. + void Release(); + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta new file mode 100644 index 000000000..2aded9e5b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 88d3c4e6b86bb4441b21a9c7f9f2a2f7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs new file mode 100644 index 000000000..acdcb32cf --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs @@ -0,0 +1,21 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides whether a uGUI hit or a UI Toolkit hit at the same point takes the input, the way the + /// EventSystem orders them for real input. + /// + internal static class MouseUiHitPriority + { + // The EventSystem compares hits from different raycasters by camera depth only when both have a camera, + // and the UI Toolkit raycaster has none; then by sortOrderPriority, where a panel reports its sorting + // order rounded down; then by renderOrderPriority, where a panel reports nearly int.MaxValue. So uGUI is + // in front only when its sortOrderPriority is strictly greater. + internal static bool IsUguiInFront(int uguiSortOrderPriority, float panelSortingOrder) + { + return uguiSortOrderPriority > Mathf.FloorToInt(panelSortingOrder); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta new file mode 100644 index 000000000..3f23b4481 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9f8021d3f2b704b10a0a03913c7c54bf +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs new file mode 100644 index 000000000..86084e7cb --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs @@ -0,0 +1,26 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// What a simulated press hit, captured as plain strings so responses never read UnityEngine objects or + /// UI Toolkit elements after an await. + /// + internal sealed class MouseUiHitReport + { + internal MouseUiHitReport(string? name, string? path, string? uiSystem) + { + Name = name; + Path = path; + UiSystem = uiSystem; + } + + internal static MouseUiHitReport None { get; } = new MouseUiHitReport(null, null, null); + + // Null means nothing was hit; the wire contract of HitGameObjectName depends on that. + internal string? Name { get; } + internal string? Path { get; } + internal string? UiSystem { get; } + internal bool HasHit => Name != null; + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta new file mode 100644 index 000000000..527384fbe --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 31881b37cb52e49c2a3ad40d2075254f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs new file mode 100644 index 000000000..d749af200 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs @@ -0,0 +1,16 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The UiSystem values simulate-mouse-ui reports, matching the ones screenshot annotations use. + /// + internal static class MouseUiHitSystem + { + internal const string UGUI = "UGUI"; + internal const string UIToolkit = "UIToolkit"; + + // A target outside any Canvas, such as an object hit by a physics raycaster. + internal const string NonUi = ""; + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta new file mode 100644 index 000000000..099173acd --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6bf90b44c9012482a8ddbd660976fd13 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs index 2dedc3bd7..26b918ea0 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs @@ -51,7 +51,9 @@ internal static ResolvedPointerTargets ResolvePressablePointerTargets( return ResolveBypassPressablePointerTargets(parameters, inputPos, pointerData, action); } - RaycastResult? hit = UiRaycastHelper.RaycastUI(screenPos, eventSystem); + // UI Toolkit is resolved separately at the requested point; the bridge's panel hits are flipped with + // Screen.height and would land on the wrong element. + RaycastResult? hit = UiRaycastHelper.RaycastUISkippingUIToolkitPanels(screenPos, eventSystem); if (hit == null) { return ResolvedPointerTargets.Empty; diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs index 116b3786c..1587e24bf 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs @@ -17,22 +17,21 @@ internal static class MouseUiPressActionExecutor { internal static async Task ExecuteClick( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, MouseUiMainThreadCleanupScheduler cleanupScheduler, CancellationToken ct) { Vector2 inputPos = new(parameters.X, parameters.Y); - Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); - PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); - MouseUiInputSystemSync.SyncMousePosition(screenPos); - ResolvedPointerTargets resolvedTargets = - MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, MouseAction.Click); - if (resolvedTargets.FailureResponse != null) + (IMouseUiPressTarget? resolvedTarget, SimulateMouseUiResponse? failure) = + ResolvePressTarget(parameters, eventSystem, inputPos, MouseAction.Click); + if (failure != null) { - return resolvedTargets.FailureResponse; + return failure; } - if (parameters.BypassRaycast && resolvedTargets.Target == null) + IMouseUiPressTarget pressTarget = resolvedTarget!; + MouseUiHitReport report = pressTarget.Report; + if (parameters.BypassRaycast && !report.HasHit) { return new SimulateMouseUiResponse { @@ -44,8 +43,7 @@ internal static async Task ExecuteClick( }; } - string? targetName = resolvedTargets.Target?.name; - bool hitTarget = resolvedTargets.Target != null; + string? targetName = report.Name; SimulateMouseUiOverlayState.Update( MouseAction.Click, inputPos, null, Handles.GetMainGameViewSize()); @@ -66,13 +64,13 @@ internal static async Task ExecuteClick( await MainThreadSwitcher.SwitchToMainThread(ct); // Fire click events after expand animation so the user sees where the click lands - ExecutePointerClickEvents(resolvedTargets, pointerData); + pressTarget.Click(); MouseUiFrameWaitOutcome dissipateOutcome = await MouseUiOverlayAnimator.PlayDissipateAnimation(ct).ConfigureAwait(false); if (dissipateOutcome == MouseUiFrameWaitOutcome.TimedOut) { cleanupScheduler.QueueOverlayClear(); - return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, report); } if (dissipateOutcome == MouseUiFrameWaitOutcome.Paused) { @@ -83,12 +81,12 @@ internal static async Task ExecuteClick( } await MainThreadSwitcher.SwitchToMainThread(ct); - return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, report); } internal static async Task ExecuteLongPress( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, MouseUiMainThreadCleanupScheduler cleanupScheduler, CancellationToken ct) { @@ -99,17 +97,16 @@ internal static async Task ExecuteLongPress( } Vector2 inputPos = new(parameters.X, parameters.Y); - Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); - PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); - MouseUiInputSystemSync.SyncMousePosition(screenPos); - ResolvedPointerTargets resolvedTargets = - MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, MouseAction.LongPress); - if (resolvedTargets.FailureResponse != null) + (IMouseUiPressTarget? resolvedTarget, SimulateMouseUiResponse? failure) = + ResolvePressTarget(parameters, eventSystem, inputPos, MouseAction.LongPress); + if (failure != null) { - return resolvedTargets.FailureResponse; + return failure; } - if (parameters.BypassRaycast && resolvedTargets.Target == null) + IMouseUiPressTarget pressTarget = resolvedTarget!; + MouseUiHitReport report = pressTarget.Report; + if (parameters.BypassRaycast && !report.HasHit) { return new SimulateMouseUiResponse { @@ -121,9 +118,7 @@ internal static async Task ExecuteLongPress( }; } - string? targetName = resolvedTargets.Target?.name; - bool hitTarget = resolvedTargets.Target != null; - bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null; + string? targetName = report.Name; SimulateMouseUiOverlayState.Update( MouseAction.LongPress, inputPos, null, Handles.GetMainGameViewSize()); @@ -143,10 +138,11 @@ internal static async Task ExecuteLongPress( } await MainThreadSwitcher.SwitchToMainThread(ct); - ExecuteLongPressPointerDown(resolvedTargets, pointerData); - try { + // Inside the try: a UI Toolkit handler that throws on the press still gets the release, which is + // safe to send when nothing was pressed. + pressTarget.PressDown(); SimulateMouseUiResponse? holdAbort = await HoldLongPressOrAbortAsync( parameters.Duration, inputPos, targetName, cleanupScheduler, ct).ConfigureAwait(false); if (holdAbort != null) @@ -157,11 +153,7 @@ internal static async Task ExecuteLongPress( finally { // Ensure pointerUp fires even if the hold loop is cancelled - if (shouldReleasePointer) - { - cleanupScheduler.ExecuteCleanupOnMainThread( - () => ExecuteEvents.Execute(resolvedTargets.Target!, pointerData, ExecuteEvents.pointerUpHandler)); - } + cleanupScheduler.ExecuteCleanupOnMainThread(pressTarget.Release); } // Why switch after the helper: HoldLongPressOrAbortAsync ends on the main thread, @@ -172,7 +164,7 @@ internal static async Task ExecuteLongPress( if (dissipateOutcome == MouseUiFrameWaitOutcome.TimedOut) { cleanupScheduler.QueueOverlayClear(); - return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, report); } if (dissipateOutcome == MouseUiFrameWaitOutcome.Paused) { @@ -183,7 +175,7 @@ internal static async Task ExecuteLongPress( } await MainThreadSwitcher.SwitchToMainThread(ct); - return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, report); } private static SimulateMouseUiResponse? CreateInvalidLongPressDurationResponse(float duration) @@ -249,53 +241,15 @@ internal static async Task ExecuteLongPress( return null; } - private static void ExecutePointerClickEvents( - ResolvedPointerTargets resolvedTargets, - PointerEventData pointerData) - { - if (resolvedTargets.RawTarget == null) - { - return; - } - - if (resolvedTargets.PressTarget != null) - { - ExecuteEvents.ExecuteHierarchy( - resolvedTargets.RawTarget, - pointerData, - ExecuteEvents.pointerDownHandler); - } - - if (resolvedTargets.Target != null) - { - ExecuteEvents.Execute( - resolvedTargets.Target, - pointerData, - ExecuteEvents.pointerUpHandler); - } - - if (resolvedTargets.ClickTarget != null) - { - ExecuteEvents.Execute( - resolvedTargets.ClickTarget, - pointerData, - ExecuteEvents.pointerClickHandler); - } - } - - private static void ExecuteLongPressPointerDown( - ResolvedPointerTargets resolvedTargets, - PointerEventData pointerData) + private static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) ResolvePressTarget( + MouseUiSimulationCommand parameters, + EventSystem? eventSystem, + Vector2 inputPos, + MouseAction action) { - if (resolvedTargets.RawTarget == null || resolvedTargets.Target == null) - { - return; - } - - ExecuteEvents.ExecuteHierarchy( - resolvedTargets.RawTarget, - pointerData, - ExecuteEvents.pointerDownHandler); + Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); + MouseUiInputSystemSync.SyncMousePosition(screenPos); + return MouseUiPressTargetResolver.Resolve(parameters, eventSystem, inputPos, screenPos, action); } } } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs new file mode 100644 index 000000000..718bab3b9 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs @@ -0,0 +1,102 @@ +#nullable enable +using UnityEngine; +using UnityEngine.EventSystems; + +using io.github.hatayama.UnityCliLoop.Runtime; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides whether a click or long press goes to uGUI or to UI Toolkit. On success a target is always + /// returned, an empty uGUI one when nothing is under the point; on failure only the response is. + /// + internal static class MouseUiPressTargetResolver + { + private const string NoEventSystemMessage = "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."; + private const string UIToolkitPathHint = + " UI Toolkit element paths are not accepted here; click the element at its SimX/SimY without --bypass-raycast."; + + internal static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) Resolve( + MouseUiSimulationCommand parameters, + EventSystem? eventSystem, + Vector2 inputPos, + Vector2 screenPos, + MouseAction action) + { + if (parameters.BypassRaycast) + { + return ResolveBypass(parameters, eventSystem, inputPos, screenPos, action); + } + + // UI Toolkit is looked up at the requested point itself, which does not depend on Screen; the uGUI + // side leaves out the bridge's panel hits, which do. + UIToolkitPointerTarget? toolkit = UIToolkitScreenElements.FindPointerTarget(inputPos); + ResolvedPointerTargets uguiResolved = ResolvedPointerTargets.Empty; + PointerEventData? uguiPointerData = null; + if (eventSystem != null) + { + uguiPointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); + uguiResolved = MouseUiPointerTargetResolver.ResolvePressablePointerTargets( + parameters, eventSystem, inputPos, screenPos, uguiPointerData, action); + } + + // A raycast hit without a handler still counts: real input stops at it and never reaches the panel. + bool uguiHasRaycastHit = uguiResolved.RawTarget != null; + if (toolkit == null && !uguiHasRaycastHit) + { + if (eventSystem == null) + { + return (null, MouseUiSimulationResponseFactory.CreateFailure( + parameters, + $"No EventSystem found in the scene, and no UI Toolkit element is at ({inputPos.x:F1}, {inputPos.y:F1}). Ensure an EventSystem GameObject exists for uGUI.")); + } + + return (new UguiPressTarget(ResolvedPointerTargets.Empty, uguiPointerData), null); + } + + if (toolkit != null && (!uguiHasRaycastHit || !IsUguiHitInFront(uguiPointerData!, toolkit))) + { + return (new UIToolkitPressTarget(toolkit, inputPos, (int)parameters.Button), null); + } + + return (new UguiPressTarget(uguiResolved, uguiPointerData), null); + } + + private static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) ResolveBypass( + MouseUiSimulationCommand parameters, + EventSystem? eventSystem, + Vector2 inputPos, + Vector2 screenPos, + MouseAction action) + { + if (eventSystem == null) + { + return (null, MouseUiSimulationResponseFactory.CreateFailure(parameters, NoEventSystemMessage)); + } + + PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); + ResolvedPointerTargets resolved = MouseUiPointerTargetResolver.ResolvePressablePointerTargets( + parameters, eventSystem, inputPos, screenPos, pointerData, action); + SimulateMouseUiResponse? failure = resolved.FailureResponse; + if (failure == null) + { + return (new UguiPressTarget(resolved, pointerData), null); + } + + // Annotation paths of UI Toolkit controls contain '#', which no GameObject path lookup can match. + if (failure.Message.EndsWith("was not found.", System.StringComparison.Ordinal) && parameters.TargetPath.Contains("#")) + { + failure.Message += UIToolkitPathHint; + } + + return (null, failure); + } + + // The press data carries the uGUI raycast hit, whose raycaster decides the order against the panel. + private static bool IsUguiHitInFront(PointerEventData uguiPointerData, UIToolkitPointerTarget toolkit) + { + BaseRaycaster raycaster = uguiPointerData.pointerPressRaycast.module; + return MouseUiHitPriority.IsUguiInFront(raycaster.sortOrderPriority, toolkit.SortingOrder); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta new file mode 100644 index 000000000..5cc46dbef --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 94dd6bfdafc504163afebc155ba0247d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs index 32bf26260..4eb4db168 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs @@ -66,19 +66,21 @@ internal static SimulateMouseUiResponse CreateFrameTimeoutResult( internal static SimulateMouseUiResponse CreateClickResult( MouseUiSimulationCommand parameters, Vector2 inputPos, - string? targetName, - bool hitTarget) + MouseUiHitReport hit) { + string? targetName = hit.Name; return new SimulateMouseUiResponse { Success = true, - Message = hitTarget + Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}'" - : $"Clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1})" + : $"Clicked {DescribeHit(hit)} at ({inputPos.x:F1}, {inputPos.y:F1})" : $"Clicked at ({inputPos.x:F1}, {inputPos.y:F1}) - no UI element hit", Action = MouseAction.Click.ToString(), HitGameObjectName = targetName, + UiSystem = hit.UiSystem, + HitPath = hit.Path, PositionX = inputPos.x, PositionY = inputPos.y }; @@ -87,24 +89,33 @@ internal static SimulateMouseUiResponse CreateClickResult( internal static SimulateMouseUiResponse CreateLongPressResult( MouseUiSimulationCommand parameters, Vector2 inputPos, - string? targetName, - bool hitTarget) + MouseUiHitReport hit) { + string? targetName = hit.Name; return new SimulateMouseUiResponse { Success = true, - Message = hitTarget + Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}' for {parameters.Duration:F1}s" - : $"Long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s" + : $"Long-pressed {DescribeHit(hit)} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s" : $"Long-pressed at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s - no UI element hit", Action = MouseAction.LongPress.ToString(), HitGameObjectName = targetName, + UiSystem = hit.UiSystem, + HitPath = hit.Path, PositionX = inputPos.x, PositionY = inputPos.y }; } + private static string DescribeHit(MouseUiHitReport hit) + { + return hit.UiSystem == MouseUiHitSystem.UIToolkit + ? $"UI Toolkit element '{hit.Name}'" + : $"'{hit.Name}'"; + } + internal static SimulateMouseUiResponse CreateDragResult( MouseUiSimulationCommand parameters, Vector2 inputStart, diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs index 239600f46..4417ed508 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs @@ -1,6 +1,4 @@ #nullable enable -using UnityEngine.EventSystems; - using io.github.hatayama.UnityCliLoop.Runtime; namespace io.github.hatayama.UnityCliLoop.FirstPartyTools @@ -12,7 +10,6 @@ internal static class MouseUiSimulationValidator { internal static SimulateMouseUiResponse? ValidateSimulationStart( MouseUiSimulationCommand parameters, - EventSystem? eventSystem, string pausedActionDescription) { PlayModeToolPreflightResult playModeResult = @@ -22,13 +19,6 @@ internal static class MouseUiSimulationValidator return MouseUiSimulationResponseFactory.CreatePreflightFailure(parameters, playModeResult); } - if (eventSystem == null) - { - return MouseUiSimulationResponseFactory.CreateFailure( - parameters, - "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."); - } - return ValidateSimulationRequestOptions(parameters); } @@ -91,7 +81,7 @@ private static bool RequiresBypassTargetPath(MouseAction action) || action == MouseAction.DragStart; } - private static bool IsDragAction(MouseAction action) + internal static bool IsDragAction(MouseAction action) { return action == MouseAction.Drag || action == MouseAction.DragStart diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs index 1c7354dc7..a4127095c 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs @@ -13,7 +13,24 @@ public class SimulateMouseUiResponse : UnityCliLoopToolResponse { public string Message { get; set; } = ""; public string Action { get; set; } = ""; + /// + /// Name of what received the press: the uGUI or physics GameObject that handles it, or the UI Toolkit + /// control under the point (its name, or its type name when it has none). Null when nothing was hit. + /// public string? HitGameObjectName { get; set; } + + /// + /// Which UI system was hit: "UGUI" for a target under a Canvas, "" for a target outside any Canvas (a + /// physics hit), "UIToolkit" for a UI Toolkit control, or null when nothing was hit. + /// + public string? UiSystem { get; set; } + + /// + /// Path of what was hit, in the form screenshot annotations use: the GameObject's hierarchy path for + /// uGUI and physics, or the document path, '#', and element chain for UI Toolkit. Null when nothing was + /// hit or when a UI Toolkit element belongs to no document, such as an open dropdown's menu. + /// + public string? HitPath { get; set; } public float PositionX { get; set; } public float PositionY { get; set; } public float? EndPositionX { get; set; } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs index 1a711654f..948a55842 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs @@ -50,14 +50,20 @@ public async Task ExecuteAsync( EventSystem? eventSystem = EventSystem.current; SimulateMouseUiResponse? validationFailure = MouseUiSimulationValidator.ValidateSimulationStart( parameters, - eventSystem, PausedActionDescription); if (validationFailure != null) { return validationFailure; } - Debug.Assert(eventSystem != null, "ValidateSimulationStart must reject a missing EventSystem."); - EventSystem activeEventSystem = eventSystem!; + + // Clicks and long presses can reach UI Toolkit without an EventSystem and check for one when they + // resolve their target. DragMove and DragEnd executors never receive it, so drags are checked here. + if (MouseUiSimulationValidator.IsDragAction(parameters.Action) && eventSystem == null) + { + return MouseUiSimulationResponseFactory.CreateFailure( + parameters, + "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."); + } MouseUiSimulationActivityLogger.LogSimulationStart(parameters, correlationId); OverlayCanvasFactory.EnsureExists(); @@ -70,7 +76,7 @@ public async Task ExecuteAsync( } SimulateMouseUiResponse response = - await ExecuteMouseAction(parameters, activeEventSystem, ct).ConfigureAwait(false); + await ExecuteMouseAction(parameters, eventSystem, ct).ConfigureAwait(false); MouseUiSimulationActivityLogger.LogSimulationComplete(parameters, response, correlationId); return response; @@ -78,7 +84,7 @@ public async Task ExecuteAsync( private async Task ExecuteMouseAction( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, CancellationToken ct) { switch (parameters.Action) @@ -91,16 +97,18 @@ private async Task ExecuteMouseAction( ct).ConfigureAwait(false); case MouseAction.Drag: + Debug.Assert(eventSystem != null, "Drags without an EventSystem must be rejected before dispatch."); return await MouseUiOneShotDragExecutor.ExecuteDragOneShot( parameters, - eventSystem, + eventSystem!, _mainThreadCleanupScheduler, ct).ConfigureAwait(false); case MouseAction.DragStart: + Debug.Assert(eventSystem != null, "Drags without an EventSystem must be rejected before dispatch."); return await MouseUiIncrementalDragExecutor.ExecuteDragStart( parameters, - eventSystem, + eventSystem!, _mainThreadCleanupScheduler, ct).ConfigureAwait(false); diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md index 03417c971..0be9c8c4f 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/SKILL.md @@ -1,7 +1,7 @@ --- name: uloop-simulate-mouse-ui toolName: simulate-mouse-ui -description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." +description: "Simulate PlayMode UI mouse actions on uGUI and UI Toolkit using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots." --- # Task @@ -64,7 +64,7 @@ uloop simulate-mouse-ui --action --x --y [options] - Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions - `--bypass-raycast` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path` - If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate -- Not for UI Toolkit entries (`UiSystem` `UIToolkit`); see `references/output.md`. +- UI Toolkit entries (`UiSystem` `UIToolkit`) can be clicked and long-pressed at `SimX`/`SimY`; `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObjects only. Details: `references/ui-toolkit.md`. - Device Simulator play view is supported. Prefer `uloop screenshot --capture-mode rendering --annotate-elements` for coordinates; they use the simulated device resolution (`Handles.GetMainGameViewSize()` / `Screen`), not the Simulator chrome scale. ## Pause Point Inspection (Standard for E2E) @@ -86,9 +86,6 @@ uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 # Long-press a button for 3 seconds uloop simulate-mouse-ui --action LongPress --x 400 --y 300 --duration 3.0 -# One-shot drag (start to end in one call) -uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 - # Slow drag for visual inspection uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --drag-speed 200 @@ -102,14 +99,14 @@ uloop simulate-mouse-ui --action DragEnd --x 600 --y 300 ## Prerequisites - Unity must be in **PlayMode** -- Target scene must have an **EventSystem** GameObject -- UI elements must have a **GraphicRaycaster** on their Canvas +- uGUI targets need an **EventSystem** and a **GraphicRaycaster** on their Canvas; UI Toolkit targets need neither - If you need runtime mouse input rather than UI pointer events, `simulate-mouse-input` assumes the project uses the New Input System; otherwise prefer `execute-dynamic-code` ## Output The response reports `Success`, `Message`, `Action`, `HitGameObjectName`, the used -coordinates, and `InterruptedByPausePoint`/`PausePointHits`. Two readings that matter: +coordinates, and `InterruptedByPausePoint`/`PausePointHits`; `UiSystem` and `HitPath` say +which UI system was hit and give its path in the annotation's form. Two readings that matter: `Click`/`LongPress` on empty space still return `Success = true` with `HitGameObjectName = null` (drag actions on empty space fail), and on a pause-point interruption `Message` states whether the pointer event fired before the pause. diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/output.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/output.md index 043b24d6a..af20d8c00 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/output.md +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/output.md @@ -5,7 +5,9 @@ Returns JSON with: - `Success`: Whether the operation succeeded - `Message`: Status message (e.g. "Hit element: ButtonStart" or "No UI element under (x, y)") - `Action`: Echoes which action was executed (`Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, or `LongPress`) -- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself +- `HitGameObjectName`: Name of the GameObject that received the pointer events (nullable string; null if nothing was hit). It is the event-handler target resolved up the hierarchy from the raycast hit (the first `IPointerDownHandler`, else the first `IPointerClickHandler`, for `Click`/`LongPress`; the first `IDragHandler` for drags), so it can be an ancestor of the topmost raycast hit — a Button's child Text reports the Button. With `--bypass-raycast` it is the handler resolved the same way from `--target-path` (also possibly an ancestor), not the raycast hit and not necessarily the raw path GameObject itself. For a UI Toolkit hit it is the control's `name`, the same as the annotation's `Name` +- `UiSystem`: Which UI system received the press (nullable string; `Click`/`LongPress` only, null if nothing was hit): `UGUI` for a target under a Canvas, `UIToolkit` for a UI Toolkit control, and `""` for a non-UI target such as a physics object +- `HitPath`: Path of the hit target in the form `AnnotatedElements[].Path` uses (nullable string; `Click`/`LongPress` only): a hierarchy path for GameObjects, `Document/Path#element` for UI Toolkit controls - `PositionX`: Target X coordinate that was used - `PositionY`: Target Y coordinate that was used - `EndPositionX`: Drag end X coordinate (nullable float; populated for drag actions only) @@ -18,4 +20,4 @@ Note: Click and LongPress on empty space (no UI element) still return `Success = ## UI Toolkit Elements -`simulate-mouse-ui` drives the uGUI EventSystem raycast path, so it does not reach UI Toolkit controls (`AnnotatedElements` entries with `UiSystem` `UIToolkit`): a click at their `SimX`/`SimY` lands on the panel's own GameObject instead. `--target-path` does not accept their `Path` either. Click them with `simulate-mouse-input` instead, which works when the scene has an EventSystem with an Input System UI input module. +`Click` and `LongPress` reach UI Toolkit controls at their annotated `SimX`/`SimY`. How presses are delivered and what is not supported: `ui-toolkit.md`. diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md new file mode 100644 index 000000000..ec3bb79d4 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md @@ -0,0 +1,46 @@ +# simulate-mouse-ui and UI Toolkit + +`Click` and `LongPress` reach UI Toolkit controls on screen-space panels — the +`AnnotatedElements` entries with `UiSystem` `UIToolkit`. Pass the entry's `SimX`/`SimY` as +`--x`/`--y`; the response reports `UiSystem` `UIToolkit`, the control's `Name` as +`HitGameObjectName`, and its annotation `Path` as `HitPath`. + +## How a press is delivered + +- The point is mapped to each panel the same way the screenshot annotation maps controls to + `SimX`/`SimY`, so an annotated point lands on the annotated control. It does not depend on + which Editor view was drawn last. +- Panels are searched front to back by their panel settings' `sortingOrder`; the first panel + with a control under the point gets the press. The reported control is the outermost one an + annotation would list, so a click on a scrollbar's thumb reports the scrollbar. +- Where a uGUI element and a UI Toolkit panel are both under the point, the uGUI element wins + only when its Canvas sorts above the panel (raycaster sort priority greater than the panel's + `sortingOrder`), the same rule the EventSystem applies to real input. A uGUI image without + any pointer handler in front still blocks the panel, as real input would. +- A full-screen container left at the default `pickingMode` (`Position`) counts as a hit, so + uGUI underneath it is not reached — real input behaves the same way. +- UI Toolkit presses do not need an EventSystem. With neither an EventSystem nor a UI Toolkit + control under the point, the command fails and says so. +- A disabled control receives the press but does not react, as with real input. + +## LongPress + +A UI Toolkit `Button` fires `clicked` once when the long press is released (UI Toolkit's +`Clickable` behaviour); a uGUI long press sends no click. A `RepeatButton` fires repeatedly +while held. + +## Not supported + +- `--bypass-raycast`, `--target-path`, and `--drop-target-path` take GameObject paths only; + a UI Toolkit `Path` (it contains `#`) is reported as not found. +- Drag actions go through the uGUI EventSystem only. +- World-space panels (Unity 6), as in the annotation. +- Two panels with the same `sortingOrder` overlapping at the point: Unity orders them by + creation order, which is not exposed, so either one may be chosen. + +## Notes + +- Move the real mouse cursor out of the Game View while simulating. Real pointer input over + the Game View reaches the same panels and can interleave with the simulated press. +- Device Simulator has the same limits as the annotation: coordinates use the simulated device + resolution. diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md.meta new file mode 100644 index 000000000..001f31d93 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/Skill/references/ui-toolkit.md.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 3aab865ee97bb4a55b610c7b9d4ac797 +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs new file mode 100644 index 000000000..b56fed501 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs @@ -0,0 +1,50 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Delivers a click or long press to a UI Toolkit control by sending pointer events to its panel at the + /// requested Game View point. + /// + internal sealed class UIToolkitPressTarget : IMouseUiPressTarget + { + private readonly UIToolkitPointerTarget _target; + private readonly Vector2 _gameViewTopLeft; + private readonly int _button; + + internal UIToolkitPressTarget(UIToolkitPointerTarget target, Vector2 gameViewTopLeft, int button) + { + _target = target; + _gameViewTopLeft = gameViewTopLeft; + _button = button; + Report = new MouseUiHitReport(target.Name, target.Path, MouseUiHitSystem.UIToolkit); + } + + public MouseUiHitReport Report { get; } + + // UI Toolkit lets handler exceptions reach the sender, unlike uGUI's ExecuteEvents, so the release is + // sent even when the press throws, and the exception still reaches the caller. + public void Click() + { + try + { + _target.PressDown(_gameViewTopLeft, _button); + } + finally + { + _target.Release(_gameViewTopLeft, _button); + } + } + + public void PressDown() + { + _target.PressDown(_gameViewTopLeft, _button); + } + + public void Release() + { + _target.Release(_gameViewTopLeft, _button); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta new file mode 100644 index 000000000..727772e4b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2a98e11783c0f44ebaaffe60ea0219d2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs new file mode 100644 index 000000000..ada219b6d --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs @@ -0,0 +1,101 @@ +#nullable enable +using UnityEngine; +using UnityEngine.EventSystems; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Delivers a click or long press to uGUI or physics targets through the EventSystem's handlers. With no + /// resolved target it reports no hit and sends nothing. + /// + internal sealed class UguiPressTarget : IMouseUiPressTarget + { + private readonly ResolvedPointerTargets _resolvedTargets; + private readonly PointerEventData? _pointerData; + private readonly bool _shouldReleasePointer; + + internal UguiPressTarget(ResolvedPointerTargets resolvedTargets, PointerEventData? pointerData) + { + _resolvedTargets = resolvedTargets; + _pointerData = pointerData; + + // Captured as plain values: Release can run from cleanup after a timeout, where checking the + // UnityEngine.Object targets for null is not safe. + bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null; + _shouldReleasePointer = shouldReleasePointer; + Report = CreateReport(resolvedTargets.Target); + } + + public MouseUiHitReport Report { get; } + + // Decided by whether the handler sits under a Canvas rather than by raycaster type, because a bypass + // press has no raycaster and both paths must report alike. + private static MouseUiHitReport CreateReport(GameObject? target) + { + if (target == null) + { + return MouseUiHitReport.None; + } + + string uiSystem = target.GetComponentInParent() != null + ? MouseUiHitSystem.UGUI + : MouseUiHitSystem.NonUi; + return new MouseUiHitReport(target.name, GameObjectPathUtility.GetFullPath(target), uiSystem); + } + + public void Click() + { + if (_resolvedTargets.RawTarget == null) + { + return; + } + + if (_resolvedTargets.PressTarget != null) + { + ExecuteEvents.ExecuteHierarchy( + _resolvedTargets.RawTarget, + _pointerData, + ExecuteEvents.pointerDownHandler); + } + + if (_resolvedTargets.Target != null) + { + ExecuteEvents.Execute( + _resolvedTargets.Target, + _pointerData, + ExecuteEvents.pointerUpHandler); + } + + if (_resolvedTargets.ClickTarget != null) + { + ExecuteEvents.Execute( + _resolvedTargets.ClickTarget, + _pointerData, + ExecuteEvents.pointerClickHandler); + } + } + + public void PressDown() + { + if (_resolvedTargets.RawTarget == null || _resolvedTargets.Target == null) + { + return; + } + + ExecuteEvents.ExecuteHierarchy( + _resolvedTargets.RawTarget, + _pointerData, + ExecuteEvents.pointerDownHandler); + } + + public void Release() + { + if (!_shouldReleasePointer) + { + return; + } + + ExecuteEvents.Execute(_resolvedTargets.Target!, _pointerData, ExecuteEvents.pointerUpHandler); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta new file mode 100644 index 000000000..8dd0ea9ee --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d99bca934334f489cb54b33b2dee12de +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef index 23bb2f8f5..4c9e61822 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef @@ -6,6 +6,7 @@ "GUID:fc3fd32eddbee40e39c2d76dc184957b", "GUID:f1a38fcd7e85047c5a86331a3e37ddfc", "GUID:a2d87883023de4cf59b7d1962d5dd8aa", + "GUID:569bba870e5e64d0d94144057b927001", "GUID:aa7cf56cc5f074e57ba35272b877e116", "GUID:c956a21f824994ef087b6de566690b3d", "GUID:33fef506e6744f2982e8c13b1196f696", diff --git a/cli/common/tools/default-tools.json b/cli/common/tools/default-tools.json index e82d40a13..8711367b6 100644 --- a/cli/common/tools/default-tools.json +++ b/cli/common/tools/default-tools.json @@ -521,7 +521,7 @@ }, { "name": "simulate-mouse-ui", - "description": "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots.", + "description": "Simulate PlayMode UI mouse actions on uGUI and UI Toolkit using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots.", "inputSchema": { "type": "object", "properties": { diff --git a/scripts/test-screenshot-uitoolkit-annotation.ps1 b/scripts/test-screenshot-uitoolkit-annotation.ps1 index db01ae543..a5371d6d4 100644 --- a/scripts/test-screenshot-uitoolkit-annotation.ps1 +++ b/scripts/test-screenshot-uitoolkit-annotation.ps1 @@ -1,7 +1,8 @@ <# -E2E verification that screenshot --annotate-elements lists UI Toolkit controls, using the uloop CLI against -UIToolkitAnnotationDemoScene. Safe to run repeatedly: it stops PlayMode and restores the Game View size on exit, -and never saves the scene. +E2E verification that screenshot --annotate-elements lists UI Toolkit controls and that simulate-mouse-ui and +simulate-mouse-input press them at the annotated points, using the uloop CLI against UIToolkitAnnotationDemoScene. +Safe to run repeatedly: it stops PlayMode and restores the Game View size on exit, and never saves the scene, so +the listener and EventSystem changes made in PlayMode are discarded. Usage: powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\test-screenshot-uitoolkit-annotation.ps1 @@ -376,7 +377,8 @@ function Test-ElementsOnlyCapture { ) [object[]]$first = Get-UiToolkitElements -Response $AnnotatedResponse - [object[]]$second = Get-UiToolkitElements -Response (Invoke-ElementsOnlyCapture) + [pscustomobject]$elementsOnly = Invoke-ElementsOnlyCapture + [object[]]$second = Get-UiToolkitElements -Response $elementsOnly [string]$firstPaths = (@($first | ForEach-Object { $_.Path }) | Sort-Object) -join "|" [string]$secondPaths = (@($second | ForEach-Object { $_.Path }) | Sort-Object) -join "|" Assert-EqualText -Actual $secondPaths -Expected $firstPaths -Context "UI Toolkit element paths in the elements-only capture" @@ -387,48 +389,219 @@ function Test-ElementsOnlyCapture { Assert-Near -Actual $element.$field -Expected $match.$field -Context "$($element.Path) $field" } } + + return $elementsOnly } -function Get-PlayClickCount { +function Get-ClickCount { + param( + [string]$Name + ) + [string]$code = @" UIToolkitAnnotationDemo demo = UnityEngine.Object.FindObjectOfType(); if (demo == null) { return "ERROR: UIToolkitAnnotationDemo not found"; } -return demo.GetClickCount("play"); +return demo.GetClickCount("$Name"); "@ [pscustomobject]$result = Invoke-UloopJson -CommandArguments @("execute-dynamic-code", "--code", $code) - Assert-Condition -Condition ($result.Success -eq $true) -Message "Read play click count failed: $($result.ErrorMessage)" + Assert-Condition -Condition ($result.Success -eq $true) -Message "Read $Name click count failed: $($result.ErrorMessage)" [string]$count = [string]$result.Result if ($count.StartsWith("ERROR:")) { - throw "Read play click count failed: $($count.Substring(6).Trim())" + throw "Read $Name click count failed: $($count.Substring(6).Trim())" } return $count } -# simulate-mouse-input reaches UI Toolkit buttons through the EventSystem; simulate-mouse-ui does not. -function Test-Click { +function Invoke-DynamicCodeText { param( - [pscustomobject]$AnnotatedResponse + [string]$Code, + [string]$Context ) - [object[]]$elements = Get-UiToolkitElements -Response $AnnotatedResponse - [pscustomobject]$play = @($elements | Where-Object { $_.Name -eq "play" })[0] - [int]$before = [int](Get-PlayClickCount) + [pscustomobject]$result = Invoke-UloopJson -CommandArguments @("execute-dynamic-code", "--code", $Code) + Assert-Condition -Condition ($result.Success -eq $true) -Message "$Context failed: $($result.ErrorMessage)" + return [string]$result.Result +} + +function Format-Coordinate { + param( + [double]$Value + ) + + return $Value.ToString([Globalization.CultureInfo]::InvariantCulture) +} + +function Get-AnnotatedElement { + param( + [pscustomobject]$Response, + [string]$UiSystem, + [string]$Name + ) + + return @($Response.Screenshots[0].AnnotatedElements | Where-Object { $_.UiSystem -eq $UiSystem -and $_.Name -eq $Name })[0] +} + +# Returns the simulate-mouse-ui response after checking it succeeded. +function Invoke-SimulateUi { + param( + [string]$Context, + [string[]]$Arguments + ) + + [pscustomobject]$response = Invoke-UloopJson -CommandArguments (@("simulate-mouse-ui") + $Arguments) + Assert-Condition -Condition ($response.Success -eq $true) -Message "$Context failed: $($response.Message)" + return $response +} + +function Invoke-SimulateUiClick { + param( + [string]$Context, + [double]$X, + [double]$Y + ) + + return Invoke-SimulateUi -Context $Context -Arguments @("--action", "Click", "--x", (Format-Coordinate $X), "--y", (Format-Coordinate $Y)) +} + +# simulate-mouse-ui reaches the UI Toolkit button directly and reports it, including for a click inside a +# scrollbar, where it must report the scrollbar rather than the part inside it. +function Test-UiClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") + + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on play" -X $play.SimX -Y $play.SimY + Assert-EqualText -Actual $response.UiSystem -Expected "UIToolkit" -Context "simulate-mouse-ui click on play UiSystem" + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/BackDocument#play" -Context "simulate-mouse-ui click on play HitPath" + Assert-EqualText -Actual $response.HitGameObjectName -Expected "play" -Context "simulate-mouse-ui click on play HitGameObjectName" + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after simulate-mouse-ui" + + [pscustomobject]$scrollbar = @($ElementsOnly.Screenshots[0].AnnotatedElements | Where-Object { $_.UiSystem -eq "UIToolkit" -and $_.Type -eq "Scrollbar" })[0] + $response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on the scrollbar" -X $scrollbar.SimX -Y $scrollbar.SimY + Assert-EqualText -Actual $response.HitPath -Expected $scrollbar.Path -Context "simulate-mouse-ui click on the scrollbar HitPath" +} + +# simulate-mouse-input reaches UI Toolkit buttons through the EventSystem's Input System UI module. +function Test-InputClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") [pscustomobject]$response = Invoke-UloopJson -CommandArguments @( "simulate-mouse-input", "--action", "Click", "--x", - ([double]$play.SimX).ToString([Globalization.CultureInfo]::InvariantCulture), + (Format-Coordinate $play.SimX), "--y", - ([double]$play.SimY).ToString([Globalization.CultureInfo]::InvariantCulture) + (Format-Coordinate $play.SimY) ) - Assert-Condition -Condition ($response.Success -eq $true) -Message "Click play failed: $($response.Message)" + Assert-Condition -Condition ($response.Success -eq $true) -Message "simulate-mouse-input click on play failed: $($response.Message)" Start-Sleep -Seconds 1 - Assert-EqualText -Actual (Get-PlayClickCount) -Expected ([string]($before + 1)) -Context "play click count" + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after simulate-mouse-input" +} + +# The front panel takes a click on the covered part; the uncovered annotated point reaches the button. +function Test-CoveredClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$partlyCovered = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "partly-covered" + [double]$coveredX = $partlyCovered.BoundsMinX + ($partlyCovered.BoundsMaxX - $partlyCovered.BoundsMinX) * 0.3 + [double]$coveredY = ($partlyCovered.BoundsMinY + $partlyCovered.BoundsMaxY) / 2 + [string]$before = Get-ClickCount -Name "partly-covered" + + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on the covered part" -X $coveredX -Y $coveredY + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/FrontDocument#cover" -Context "simulate-mouse-ui click on the covered part HitPath" + Assert-EqualText -Actual (Get-ClickCount -Name "partly-covered") -Expected $before -Context "partly-covered click count after clicking its cover" + + Invoke-SimulateUiClick -Context "simulate-mouse-ui click on partly-covered" -X $partlyCovered.SimX -Y $partlyCovered.SimY | Out-Null + Assert-EqualText -Actual (Get-ClickCount -Name "partly-covered") -Expected ([string]([int]$before + 1)) -Context "partly-covered click count" +} + +# The disabled button is not annotated, so its layout centre in the demo (100,440 200x50) is used. The press is +# delivered but, as with real input, the disabled button does not react. +function Test-DisabledClick { + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on locked" -X 200 -Y 465 + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/BackDocument#locked" -Context "simulate-mouse-ui click on locked HitPath" + Assert-EqualText -Actual (Get-ClickCount -Name "locked") -Expected "0" -Context "locked click count" +} + +function Test-LongPress { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") + + Invoke-SimulateUi -Context "simulate-mouse-ui long press on play" -Arguments @( + "--action", "LongPress", "--duration", "0.5", "--x", (Format-Coordinate $play.SimX), "--y", (Format-Coordinate $play.SimY)) | Out-Null + + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after long press" +} + +# The demo does not count uGUI clicks, so a listener added in PlayMode writes the count into the button label. +function Test-UguiClick { + param( + [pscustomobject]$ElementsOnly + ) + + [string]$listenerCode = @' +using UnityEngine; +using UnityEngine.UI; +GameObject buttonObject = GameObject.Find("ugui-button"); +if (buttonObject == null) { return "ERROR: ugui-button not found"; } +Text label = buttonObject.GetComponentInChildren(); +if (label == null) { return "ERROR: ugui-button has no Text"; } +int clicks = 0; +buttonObject.GetComponent