From 3e5141559a91634c5a113919b856f59d6d9e4571 Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:20:36 +0900 Subject: [PATCH 1/8] refactor: keep UIToolkitScreenControl constructible only inside its module Only the module builds these results; callers read them. An internal constructor makes that a compiler rule rather than a convention. --- .../FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs index c8d69dfe2..88c49d81d 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs @@ -8,7 +8,7 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// public sealed class UIToolkitScreenControl { - public UIToolkitScreenControl( + internal UIToolkitScreenControl( string name, string path, string type, From 392201e61c87b7a0cecc7e4717d546618126e093 Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:20:49 +0900 Subject: [PATCH 2/8] feat: find the UI Toolkit control under a Game View point and send it pointer input simulate-mouse-ui needs to press UI Toolkit controls at the points screenshot annotations report. Going through the uGUI bridge does not work there: it flips Y with Screen.height, which outside the player loop is the size of whichever Editor view was drawn last. UIToolkitScreenElements.FindPointerTarget maps the point onto each panel with the same function Unity applies to input, tries panels front to back, and reports the outermost control around the picked element, so the path matches the annotation even on a scrollbar's arrow or a slider's input field. Elements outside every document, such as an open dropdown's menu, get no path. The returned target sends pointer down, move and up events to the panel without a target, so Unity delivers them to the element holding the pointer or under it, as with real input. Once the owning document is disabled or destroyed, the target sends nothing and throws nothing, because that can happen between resolving and pressing and during cleanup. --- .../UIToolkitScreenElementsPointerTests.cs | 397 ++++++++++++++++++ ...IToolkitScreenElementsPointerTests.cs.meta | 11 + .../UIToolkit/UIToolkitPointerEvents.cs | 55 +++ .../UIToolkit/UIToolkitPointerEvents.cs.meta | 11 + .../UIToolkit/UIToolkitPointerTarget.cs | 108 +++++ .../UIToolkit/UIToolkitPointerTarget.cs.meta | 11 + .../UIToolkit/UIToolkitPointerTargetFinder.cs | 82 ++++ .../UIToolkitPointerTargetFinder.cs.meta | 11 + .../UIToolkit/UIToolkitScreenElements.cs | 12 + 9 files changed, 698 insertions(+) create mode 100644 Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs create mode 100644 Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta diff --git a/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs new file mode 100644 index 000000000..f21a78079 --- /dev/null +++ b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs @@ -0,0 +1,397 @@ +using System.Collections; +using System.Collections.Generic; +using NUnit.Framework; +using UnityEditor; +using UnityEngine; +using UnityEngine.TestTools; +using UnityEngine.UIElements; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.ToolContracts; + +namespace io.github.hatayama.UnityCliLoop.Tests.PlayMode +{ + /// + /// Verifies which UI Toolkit control a Game View point resolves to on live runtime panels, and that presses, + /// moves and releases sent through the result reach the panel like real mouse input. Points are top-left + /// Game View pixels; panels use a scale of 1, so they equal panel coordinates. Every object is destroyed in + /// TearDown. + /// + public sealed class UIToolkitScreenElementsPointerTests + { + private const string Prefix = "UIToolkitScreenElementsPointerTests_"; + private const float Left = 100f; + private const float Top = 50f; + private const float Width = 200f; + private const float Height = 40f; + private const string RuntimeThemePath = "Assets/UI Toolkit/UnityThemes/UnityDefaultRuntimeTheme.tss"; + + private readonly List _createdObjects = new List(); + + [TearDown] + public void TearDown() + { + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.Destroy(createdObject); + } + } + + _createdObjects.Clear(); + } + + /// + /// Verifies that where two panels overlap, the control on the panel with the higher sorting order is + /// returned and the one behind it is not. + /// + [UnityTest] + public IEnumerator FindPointerTarget_WherePanelsOverlap_ReturnsFrontPanelControl() + { + UIDocument front = CreateDocument("Front", CreatePanelSettings(10f)); + AddPlaced(front.rootVisualElement, new Button { name = "front" }, Left, Top, Width, Height); + UIDocument back = CreateDocument("Back", CreatePanelSettings(0f)); + AddPlaced(back.rootVisualElement, new Button { name = "back" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("front")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Front#front")); + Assert.That(target.SortingOrder, Is.EqualTo(10f)); + } + + /// + /// Verifies a point on a button reports the button's name and its annotation-style path. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnButton_ReportsNameAndPath() + { + UIDocument document = CreateDocument("Doc", CreatePanelSettings(0f)); + AddPlaced(document.rootVisualElement, new Button { name = "play" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("play")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Doc#play")); + } + + /// + /// Verifies a point on a part inside a toggle still reports the toggle. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnPartInsideToggle_ReportsToggle() + { + UIDocument document = CreateDocument("Toggle", CreatePanelSettings(0f)); + Toggle toggle = new Toggle { name = "sound" }; + AddPlaced(document.rootVisualElement, toggle, Left, Top, Width, Height); + yield return WaitForLayout(); + VisualElement checkmark = toggle.Q(className: Toggle.checkmarkUssClassName); + Vector2 point = checkmark.worldBound.center; + Assert.That(document.rootVisualElement.panel.Pick(point), Is.Not.SameAs(toggle), "the point must hit a part inside the toggle"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("sound")); + Assert.That(target.Path, Is.EqualTo(Prefix + "Toggle#sound")); + } + + /// + /// Verifies points on the dragger and both arrow buttons of a scroll view's scroller all report the + /// scroller with the same path screenshot annotations list for it, not the slider or buttons inside. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnScrollerParts_ReportsScrollerWithAnnotationPath() + { + UIDocument document = CreateDocument("Scroll", CreatePanelSettings(0f)); + ScrollView scrollView = CreateScrollView(document.rootVisualElement); + yield return WaitForLayout(); + Scroller scroller = scrollView.verticalScroller; + UIElementInfo annotated = FindAnnotated("Scroll", "Scrollbar"); + VisualElement[] parts = { scroller.slider.Q("unity-dragger"), scroller.lowButton, scroller.highButton }; + + foreach (VisualElement part in parts) + { + Vector2 point = part.worldBound.center; + Assert.That(document.rootVisualElement.panel.Pick(point), Is.Not.SameAs(scroller), "the point must hit a part inside the scroller"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null, part.name); + Assert.That(target.Path, Is.EqualTo(annotated.Path), part.name); + Assert.That(target.Name, Is.EqualTo(annotated.Name), part.name); + } + } + + /// + /// Verifies a point on the input field of a slider that shows one reports the slider. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnSliderInputField_ReportsSlider() + { + UIDocument document = CreateDocument("Slider", CreatePanelSettings(0f)); + Slider slider = new Slider(0f, 1f) { name = "volume", showInputField = true }; + AddPlaced(document.rootVisualElement, slider, Left, Top, 300f, 30f); + yield return WaitForLayout(); + TextField inputField = slider.Q(); + Vector2 point = inputField.worldBound.center; + VisualElement picked = document.rootVisualElement.panel.Pick(point); + Assert.That(picked == inputField || inputField.Contains(picked), Is.True, "the point must hit the input field"); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Path, Is.EqualTo(Prefix + "Slider#volume")); + } + + /// + /// Verifies a point with no element under it returns null. + /// + [UnityTest] + public IEnumerator FindPointerTarget_OnEmptyPoint_ReturnsNull() + { + UIDocument document = CreateDocument("Empty", CreatePanelSettings(0f)); + AddPlaced(document.rootVisualElement, new Button { name = "play" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + Assert.That(UIToolkitScreenElements.FindPointerTarget(new Vector2(Left + Width + 50f, Top + Height + 50f)), Is.Null); + } + + /// + /// Verifies a front panel that renders into a texture still takes the point, as Unity still routes + /// screen input to it. + /// + [UnityTest] + public IEnumerator FindPointerTarget_FrontPanelWithTargetTexture_ReturnsItsControl() + { + UIDocument back = CreateDocument("TextureBack", CreatePanelSettings(0f)); + AddPlaced(back.rootVisualElement, new Button { name = "target" }, Left, Top, Width, Height); + PanelSettings frontSettings = CreatePanelSettings(10f); + frontSettings.targetTexture = CreateScreenSizedTexture(); + UIDocument front = CreateDocument("TextureFront", frontSettings); + AddPlaced(front.rootVisualElement, new Button { name = "cover" }, Left, Top, Width, Height); + yield return WaitForLayout(); + + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + + Assert.That(target, Is.Not.Null); + Assert.That(target.Name, Is.EqualTo("cover")); + } + + /// + /// Verifies a left press and release click a button once and leave no element holding the pointer. + /// + [UnityTest] + public IEnumerator PressDownThenRelease_OnButton_ClicksOnceAndLeavesNoCapture() + { + UIDocument document = CreateDocument("Click", CreatePanelSettings(0f)); + Button button = new Button { name = "play" }; + int clicks = 0; + button.clicked += () => clicks++; + AddPlaced(document.rootVisualElement, button, Left, Top, Width, Height); + yield return WaitForLayout(); + Vector2 point = Center(Left, Top, Width, Height); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + Assert.That(target, Is.Not.Null); + + target.PressDown(point, 0); + target.Release(point, 0); + + Assert.That(clicks, Is.EqualTo(1)); + Assert.That(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId), Is.Null); + } + + /// + /// Verifies that once the target's document is disabled the target reports itself unavailable, and + /// presses, moves and releases throw nothing and send nothing to the panel, which another document + /// still shows. + /// + [UnityTest] + public IEnumerator PointerCalls_AfterDocumentDisabled_SendNothing() + { + PanelSettings sharedSettings = CreatePanelSettings(0f); + UIDocument disabled = CreateDocument("Disabled", sharedSettings); + AddPlaced(disabled.rootVisualElement, new Button { name = "gone" }, Left, Top, Width, Height); + UIDocument remaining = CreateDocument("Remaining", sharedSettings); + Button witness = new Button { name = "witness" }; + AddPlaced(remaining.rootVisualElement, witness, Left, Top + 100f, Width, Height); + int received = 0; + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + witness.RegisterCallback(_ => received++, TrickleDown.TrickleDown); + yield return WaitForLayout(); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(Center(Left, Top, Width, Height)); + Assert.That(target, Is.Not.Null); + Vector2 witnessPoint = Center(Left, Top + 100f, Width, Height); + + disabled.enabled = false; + yield return null; + + Assert.That(target.IsAvailable, Is.False); + Assert.DoesNotThrow(() => target.PressDown(witnessPoint, 0)); + Assert.DoesNotThrow(() => target.MoveTo(witnessPoint)); + Assert.DoesNotThrow(() => target.Release(witnessPoint, 0)); + Assert.That(received, Is.EqualTo(0)); + + // The callbacks do fire when a target that is still available sends to the same point. + UIToolkitPointerTarget witnessTarget = UIToolkitScreenElements.FindPointerTarget(witnessPoint); + witnessTarget.PressDown(witnessPoint, 0); + witnessTarget.Release(witnessPoint, 0); + Assert.That(received, Is.GreaterThan(0)); + } + + /// + /// Verifies a right press and release reach the button as button 1 and do not click it. + /// + [UnityTest] + public IEnumerator PressDownThenRelease_WithRightButton_DoesNotClick() + { + UIDocument document = CreateDocument("Right", CreatePanelSettings(0f)); + Button button = new Button { name = "play" }; + int clicks = 0; + int pressedButton = -1; + button.clicked += () => clicks++; + button.RegisterCallback(evt => pressedButton = evt.button, TrickleDown.TrickleDown); + AddPlaced(document.rootVisualElement, button, Left, Top, Width, Height); + yield return WaitForLayout(); + Vector2 point = Center(Left, Top, Width, Height); + UIToolkitPointerTarget target = UIToolkitScreenElements.FindPointerTarget(point); + Assert.That(target, Is.Not.Null); + + target.PressDown(point, 1); + target.Release(point, 1); + + Assert.That(pressedButton, Is.EqualTo(1)); + Assert.That(clicks, Is.EqualTo(0)); + } + + /// + /// Verifies an item of an open dropdown menu, which belongs to no document, resolves with a null path + /// and pressing it changes the dropdown's value. + /// + [UnityTest] + public IEnumerator PressOnOpenDropdownMenuItem_ChangesValueAndHasNoPath() + { + UIDocument document = CreateDocument("Dropdown", CreatePanelSettings(0f)); + DropdownField dropdown = new DropdownField(new List { "low", "high" }, 0) { name = "quality" }; + AddPlaced(document.rootVisualElement, dropdown, Left, Top, Width, 30f); + yield return WaitForLayout(); + Vector2 dropdownPoint = Center(Left, Top, Width, 30f); + UIToolkitPointerTarget opener = UIToolkitScreenElements.FindPointerTarget(dropdownPoint); + Assert.That(opener, Is.Not.Null); + Assert.That(opener.Path, Is.EqualTo(Prefix + "Dropdown#quality")); + opener.PressDown(dropdownPoint, 0); + opener.Release(dropdownPoint, 0); + yield return WaitForLayout(); + List items = document.rootVisualElement.panel.visualTree + .Query(className: GenericDropdownMenu.itemUssClassName).ToList(); + Assert.That(items.Count, Is.EqualTo(2), "the dropdown menu must be open"); + Vector2 itemPoint = items[1].worldBound.center; + + UIToolkitPointerTarget item = UIToolkitScreenElements.FindPointerTarget(itemPoint); + + Assert.That(item, Is.Not.Null); + Assert.That(item.Path, Is.Null); + item.PressDown(itemPoint, 0); + item.Release(itemPoint, 0); + Assert.That(dropdown.value, Is.EqualTo("high")); + } + + private ScrollView CreateScrollView(VisualElement root) + { + ScrollView scrollView = new ScrollView { name = "list" }; + scrollView.verticalScrollerVisibility = ScrollerVisibility.AlwaysVisible; + scrollView.horizontalScrollerVisibility = ScrollerVisibility.Hidden; + AddPlaced(root, scrollView, Left, Top, Width, 100f); + Scroller scroller = scrollView.verticalScroller; + SetSize(scroller, 20f, 100f); + SetSize(scroller.lowButton, 20f, 20f); + SetSize(scroller.highButton, 20f, 20f); + SetSize(scroller.slider, 20f, 60f); + // The scroll view disables a scroller whose content fits, so the viewport is sized and given + // content taller than itself that does not shrink to fit. + SetSize(scrollView.contentViewport, 180f, 100f); + VisualElement content = new VisualElement { name = "content" }; + SetSize(content, 180f, 500f); + content.style.flexShrink = 0f; + scrollView.Add(content); + return scrollView; + } + + private static UIElementInfo FindAnnotated(string documentName, string type) + { + List elements = new List(); + UIToolkitElementCollector.Collect(elements, Handles.GetMainGameViewSize().y); + List matches = elements.FindAll( + element => element.Type == type && element.Path.StartsWith(Prefix + documentName + "#")); + Assert.That(matches.Count, Is.EqualTo(1), "the annotation must list exactly one " + type); + return matches[0]; + } + + private PanelSettings CreatePanelSettings(float sortingOrder) + { + PanelSettings settings = ScriptableObject.CreateInstance(); + settings.scaleMode = PanelScaleMode.ConstantPixelSize; + settings.scale = 1f; + settings.sortingOrder = sortingOrder; + + // Without a theme, the parts inside controls get no size and the dropdown menu finds no root + // container to open in, so the points these tests aim at would not exist. + settings.themeStyleSheet = AssetDatabase.LoadAssetAtPath(RuntimeThemePath); + Assert.That(settings.themeStyleSheet, Is.Not.Null, "the runtime theme must exist at " + RuntimeThemePath); + _createdObjects.Add(settings); + return settings; + } + + private RenderTexture CreateScreenSizedTexture() + { + RenderTexture texture = new RenderTexture(Screen.width, Screen.height, 0); + _createdObjects.Add(texture); + return texture; + } + + // The object starts inactive so the document sees its panel settings when it first enables. + private UIDocument CreateDocument(string name, PanelSettings settings) + { + GameObject documentObject = new GameObject(Prefix + name); + documentObject.SetActive(false); + _createdObjects.Add(documentObject); + UIDocument document = documentObject.AddComponent(); + document.panelSettings = settings; + documentObject.SetActive(true); + return document; + } + + private static void AddPlaced(VisualElement parent, VisualElement element, float left, float top, float width, float height) + { + element.style.position = Position.Absolute; + element.style.left = left; + element.style.top = top; + SetSize(element, width, height); + parent.Add(element); + } + + private static void SetSize(VisualElement element, float width, float height) + { + element.style.width = width; + element.style.height = height; + } + + private static Vector2 Center(float left, float top, float width, float height) + { + return new Vector2(left + width / 2f, top + height / 2f); + } + + // The root's size is set by the runtime panel update in the player loop, so frames must pass. + private static IEnumerator WaitForLayout() + { + yield return null; + yield return null; + } + } +} diff --git a/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta new file mode 100644 index 000000000..12888fdd0 --- /dev/null +++ b/Assets/Tests/PlayMode/UIToolkitScreenElementsPointerTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: f91b3eaa63d5945eea43a7aebdd41262 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs new file mode 100644 index 000000000..47e9bee2b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs @@ -0,0 +1,55 @@ +#nullable enable +using System; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Sends mouse pointer events to a runtime panel the way Unity turns real mouse input into them. + /// + internal static class UIToolkitPointerEvents + { + // No target is set, so the panel hands the event to the element holding the pointer or, failing that, + // the element under it, as it does for real input. GetPooled also records the press or release in the + // pointer device state, which fills pressedButtons for later events. + internal static void Send(IPanel panel, EventType type, Vector2 panelPoint, Vector2 delta, int button) + { + Event systemEvent = new Event + { + type = type, + mousePosition = panelPoint, + delta = delta, + button = button, + clickCount = type == EventType.MouseDown || type == EventType.MouseUp ? 1 : 0 + }; + + switch (type) + { + case EventType.MouseDown: + using (PointerDownEvent pointerDown = PointerDownEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerDown); + } + + return; + case EventType.MouseMove: + using (PointerMoveEvent pointerMove = PointerMoveEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerMove); + } + + return; + case EventType.MouseUp: + using (PointerUpEvent pointerUp = PointerUpEvent.GetPooled(systemEvent)) + { + panel.visualTree.SendEvent(pointerUp); + } + + return; + default: + throw new ArgumentOutOfRangeException(nameof(type), type, "Only mouse down, move and up are sent."); + } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta new file mode 100644 index 000000000..a36928167 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerEvents.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2dfc472d65b414f7ca31681699b8e0f1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs new file mode 100644 index 000000000..0a4fb7fda --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs @@ -0,0 +1,108 @@ +#nullable enable +using System; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The UI Toolkit control under a Game View point, and the panel that takes pointer input there. Presses, + /// moves and releases are sent to that panel the way real input reaches it, so they land on whichever + /// element is under the pointer or holds the pointer at that moment. + /// + public sealed class UIToolkitPointerTarget + { + private const int MaxMouseButton = 2; + + private readonly IPanel _panel; + private readonly UIDocument _document; + private Vector2 _lastPanelPoint; + + internal UIToolkitPointerTarget(IPanel panel, UIDocument document, string name, string? path, float sortingOrder) + { + if (string.IsNullOrEmpty(name)) + { + // Callers treat a null name as "nothing was hit", so a hit must always carry one. + throw new ArgumentException("A UI Toolkit pointer target needs a display name.", nameof(name)); + } + + _panel = panel ?? throw new ArgumentNullException(nameof(panel)); + _document = document != null ? document : throw new ArgumentNullException(nameof(document)); + Name = name; + Path = path; + SortingOrder = sortingOrder; + } + + /// The control's name, or its type name when it has none. Never null or empty. + public string Name { get; } + + /// + /// The same path screenshot annotations report for the control, or null when the element belongs to no + /// document, as with the menu of an open dropdown. + /// + public string? Path { get; } + + /// The sorting order of the panel the control is on, as set in its panel settings. + public float SortingOrder { get; } + + /// False once the document that owns the panel has been disabled or destroyed. + public bool IsAvailable => + _document != null && _document.isActiveAndEnabled && _document.rootVisualElement?.panel == _panel; + + /// Presses a mouse button (0 left, 1 right, 2 middle) at a Game View point. + /// Does nothing once the target is no longer available. + public void PressDown(Vector2 gameViewTopLeft, int button) + { + ValidateButton(button); + if (!IsAvailable) + { + // The document can go away between resolving the target and pressing, a frame or more later; + // uGUI likewise sends nothing to a destroyed or inactive target. + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + UIToolkitPointerEvents.Send(_panel, EventType.MouseDown, panelPoint, Vector2.zero, button); + _lastPanelPoint = panelPoint; + } + + /// Moves the pointer to a Game View point, carrying any buttons still held. + /// Does nothing once the target is no longer available. + public void MoveTo(Vector2 gameViewTopLeft) + { + if (!IsAvailable) + { + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + + // The button is overwritten with "none" for moves; held buttons travel in pressedButtons. + UIToolkitPointerEvents.Send(_panel, EventType.MouseMove, panelPoint, panelPoint - _lastPanelPoint, 0); + _lastPanelPoint = panelPoint; + } + + /// Releases a mouse button (0 left, 1 right, 2 middle) at a Game View point. + /// Does nothing once the target is no longer available, so cleanup code can always call it. + public void Release(Vector2 gameViewTopLeft, int button) + { + ValidateButton(button); + if (!IsAvailable) + { + return; + } + + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(_panel, gameViewTopLeft); + UIToolkitPointerEvents.Send(_panel, EventType.MouseUp, panelPoint, panelPoint - _lastPanelPoint, button); + _lastPanelPoint = panelPoint; + } + + private static void ValidateButton(int button) + { + if (button < 0 || button > MaxMouseButton) + { + throw new ArgumentOutOfRangeException(nameof(button), button, "Mouse button must be 0, 1 or 2."); + } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta new file mode 100644 index 000000000..12ad95392 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 71c2930e001654e8e8c062c127c171e3 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs new file mode 100644 index 000000000..07a3c911c --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs @@ -0,0 +1,82 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides which panel takes pointer input at a Game View point and which control to report for it. + /// + internal static class UIToolkitPointerTargetFinder + { + // Expects the panels' layouts to be settled already, because picking reads them. The first panel, front + // to back, that picks anything takes the input, as Unity routes it. Panels with the same sorting order + // keep the order they were gathered in, the same known limit as the annotation's cover check. + internal static UIToolkitPointerTarget? Find(List panels, Vector2 gameViewTopLeft) + { + foreach (UIToolkitScreenPanels.PanelEntry entry in panels.OrderByDescending(panel => panel.SortingOrder)) + { + VisualElement? picked = UIToolkitScreenPanels.PickInPanel(entry.Panel, gameViewTopLeft); + if (picked == null) + { + continue; + } + + return CreateTarget(entry, picked); + } + + return null; + } + + private static UIToolkitPointerTarget CreateTarget(UIToolkitScreenPanels.PanelEntry entry, VisualElement picked) + { + UIDocument? document = FindOwningDocument(entry, picked); + if (document == null) + { + // Elements outside every document, such as the menu of an open dropdown, have no annotation path. + // The panel's first document stands in to tell when the panel goes away. + return new UIToolkitPointerTarget( + entry.Panel, entry.Documents[0], UIToolkitElementPath.GetDisplayName(picked), null, entry.SortingOrder); + } + + VisualElement reported = FindOutermostControl(document.rootVisualElement, picked) ?? picked; + return new UIToolkitPointerTarget( + entry.Panel, + document, + UIToolkitElementPath.GetDisplayName(reported), + UIToolkitElementPath.Build(document, reported), + entry.SortingOrder); + } + + private static UIDocument? FindOwningDocument(UIToolkitScreenPanels.PanelEntry entry, VisualElement picked) + { + for (VisualElement? current = picked; current != null; current = current.hierarchy.parent) + { + UIDocument? document = entry.Documents.Find(candidate => candidate.rootVisualElement == current); + if (document != null) + { + return document; + } + } + + return null; + } + + // The outermost, not the nearest, control: annotations stop at the first control on the way down from + // the root and never list the parts inside it. A Scroller holds a Slider and two RepeatButtons, and a + // Slider with an input field holds a TextField; reporting those parts would not match the annotation. + private static VisualElement? FindOutermostControl(VisualElement root, VisualElement picked) + { + List chain = new List(); + for (VisualElement? current = picked; current != null && current != root; current = current.hierarchy.parent) + { + chain.Add(current); + } + + chain.Reverse(); + return chain.Find(element => UIToolkitElementClassifier.Classify(element) != null); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta new file mode 100644 index 000000000..60e91f6c6 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPointerTargetFinder.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 644b0b96772bd4ed0bfd525bc25d378f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs index f3351c571..25f57968a 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs @@ -32,5 +32,17 @@ public static IReadOnlyList CollectControls( UIToolkitScreenPanels.SettleLayouts(panels); return UIToolkitControlCollector.Collect(panels, probePointsForTopLeftBounds); } + + /// + /// Returns the control that pointer input at a Game View point reaches, or null when no panel takes + /// input there. Panels are tried front to back, as Unity routes input. The control reported is the + /// outermost control around the element under the point, the one screenshot annotations list. + /// + public static UIToolkitPointerTarget? FindPointerTarget(Vector2 gameViewTopLeft) + { + List panels = UIToolkitScreenPanels.Gather(); + UIToolkitScreenPanels.SettleLayouts(panels); + return UIToolkitPointerTargetFinder.Find(panels, gameViewTopLeft); + } } } From 4d7b95117040575533f73d45cb110af629388023 Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:24:33 +0900 Subject: [PATCH 3/8] refactor: route simulate-mouse-ui presses through a press target (behaviour unchanged) Clicks and long presses are about to reach UI Toolkit controls as well as uGUI ones. The executor now works with one IMouseUiPressTarget and the hit report it captured before the first await, so a second kind of target can slot in without new branches in the timeout, pause and cleanup paths. UguiPressTarget carries the existing uGUI dispatch unchanged: the click events, the long-press pointer down, and the pointer up in the finally, with the release decision still captured as a plain bool. The response factory takes the hit report instead of a name and a hit flag. The source guard test now checks the captured report and the release bool in their new places. The SimulateMouseUi assembly references the shared UI Toolkit module, which the next change uses. --- .../MouseUiSimulationResponseFactoryTests.cs | 9 +- .../Editor/StaticFacadeStateGuardTests.cs | 17 ++- .../SimulateMouseUi/IMouseUiPressTarget.cs | 20 ++++ .../IMouseUiPressTarget.cs.meta | 11 ++ .../SimulateMouseUi/MouseUiHitReport.cs | 26 ++++ .../SimulateMouseUi/MouseUiHitReport.cs.meta | 11 ++ .../MouseUiPressActionExecutor.cs | 111 ++++++------------ .../MouseUiSimulationResponseFactory.cs | 12 +- .../SimulateMouseUi/UguiPressTarget.cs | 85 ++++++++++++++ .../SimulateMouseUi/UguiPressTarget.cs.meta | 11 ++ ...stPartyTools.SimulateMouseUi.Editor.asmdef | 1 + 11 files changed, 225 insertions(+), 89 deletions(-) create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta diff --git a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs index f7f9ab351..a3a03654c 100644 --- a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs +++ b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs @@ -103,7 +103,7 @@ public void CreateClickResult_WithHitMode_MapsMessageAndFields( Vector2 position = new(10.25f, 20.75f); SimulateMouseUiResponse response = - MouseUiSimulationResponseFactory.CreateClickResult(command, position, targetName, hitTarget); + MouseUiSimulationResponseFactory.CreateClickResult(command, position, CreateHit(hitTarget, targetName)); Assert.That(response.Success, Is.True); Assert.That(response.Message, Is.EqualTo(expectedMessage)); @@ -136,7 +136,7 @@ public void CreateLongPressResult_WithHitMode_MapsMessageAndFields( Vector2 position = new(10.25f, 20.75f); SimulateMouseUiResponse response = - MouseUiSimulationResponseFactory.CreateLongPressResult(command, position, targetName, hitTarget); + MouseUiSimulationResponseFactory.CreateLongPressResult(command, position, CreateHit(hitTarget, targetName)); Assert.That(response.Success, Is.True); Assert.That(response.Message, Is.EqualTo(expectedMessage)); @@ -218,5 +218,10 @@ private static MouseUiSimulationCommand CreateCommand(SimulateMouseUiSchema sche Assert.That(command, Is.Not.Null); return command!; } + + private static MouseUiHitReport CreateHit(bool hitTarget, string? targetName) + { + return hitTarget ? new MouseUiHitReport(targetName, null, null) : MouseUiHitReport.None; + } } } diff --git a/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs b/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs index bbc231a34..56e2406c9 100644 --- a/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs +++ b/Assets/Tests/Editor/StaticFacadeStateGuardTests.cs @@ -355,14 +355,19 @@ public void ExecuteDynamicCodeUseCase_WhenAwaitingWorkflow_DoesNotCaptureEditorS [Test] public void MouseUiPressActionExecutor_WhenReturningAfterTimeout_UsesCapturedUnityObjectState() { - // Tests that timeout result branches use plain captured bools instead of UnityEngine.Object null checks. + // Tests that timeout result branches use the hit report captured before the first await, and that + // the release decision is a captured bool, instead of UnityEngine.Object null checks after awaits. string source = ReadSourceFile( "Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs"); - - Assert.That(source, Does.Contain("bool hitTarget = resolvedTargets.Target != null;")); - Assert.That(source, Does.Contain("bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null;")); - Assert.That(source, Does.Not.Contain("CreateClickResult(parameters, inputPos, targetName, resolvedTargets.Target != null)")); - Assert.That(source, Does.Not.Contain("CreateLongPressResult(parameters, inputPos, targetName, resolvedTargets.Target != null)")); + string uguiTargetSource = ReadSourceFile( + "Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs"); + + Assert.That(source, Does.Contain("MouseUiHitReport report = pressTarget.Report;")); + Assert.That(uguiTargetSource, Does.Contain("bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null;")); + Assert.That(uguiTargetSource, Does.Contain("if (!_shouldReleasePointer)")); + Assert.That(source, Does.Not.Contain("CreateClickResult(parameters, inputPos, pressTarget.Report)")); + Assert.That(source, Does.Not.Contain("CreateLongPressResult(parameters, inputPos, pressTarget.Report)")); + Assert.That(source, Does.Not.Contain("resolvedTargets.Target")); } [Test] diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs new file mode 100644 index 000000000..271329562 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The thing a click or long press is delivered to, whichever UI system it belongs to. + /// + internal interface IMouseUiPressTarget + { + // Captured when the target is created, so it stays valid after the target goes away. + MouseUiHitReport Report { get; } + + void Click(); + + void PressDown(); + + // Safe to call from cleanup even if nothing was pressed or the target has gone away. + void Release(); + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta new file mode 100644 index 000000000..2aded9e5b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/IMouseUiPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 88d3c4e6b86bb4441b21a9c7f9f2a2f7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs new file mode 100644 index 000000000..86084e7cb --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs @@ -0,0 +1,26 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// What a simulated press hit, captured as plain strings so responses never read UnityEngine objects or + /// UI Toolkit elements after an await. + /// + internal sealed class MouseUiHitReport + { + internal MouseUiHitReport(string? name, string? path, string? uiSystem) + { + Name = name; + Path = path; + UiSystem = uiSystem; + } + + internal static MouseUiHitReport None { get; } = new MouseUiHitReport(null, null, null); + + // Null means nothing was hit; the wire contract of HitGameObjectName depends on that. + internal string? Name { get; } + internal string? Path { get; } + internal string? UiSystem { get; } + internal bool HasHit => Name != null; + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta new file mode 100644 index 000000000..527384fbe --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitReport.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 31881b37cb52e49c2a3ad40d2075254f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs index 116b3786c..7f0755d77 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs @@ -22,17 +22,16 @@ internal static async Task ExecuteClick( CancellationToken ct) { Vector2 inputPos = new(parameters.X, parameters.Y); - Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); - PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); - MouseUiInputSystemSync.SyncMousePosition(screenPos); - ResolvedPointerTargets resolvedTargets = - MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, MouseAction.Click); - if (resolvedTargets.FailureResponse != null) + (IMouseUiPressTarget? resolvedTarget, SimulateMouseUiResponse? failure) = + ResolvePressTarget(parameters, eventSystem, inputPos, MouseAction.Click); + if (failure != null) { - return resolvedTargets.FailureResponse; + return failure; } - if (parameters.BypassRaycast && resolvedTargets.Target == null) + IMouseUiPressTarget pressTarget = resolvedTarget!; + MouseUiHitReport report = pressTarget.Report; + if (parameters.BypassRaycast && !report.HasHit) { return new SimulateMouseUiResponse { @@ -44,8 +43,7 @@ internal static async Task ExecuteClick( }; } - string? targetName = resolvedTargets.Target?.name; - bool hitTarget = resolvedTargets.Target != null; + string? targetName = report.Name; SimulateMouseUiOverlayState.Update( MouseAction.Click, inputPos, null, Handles.GetMainGameViewSize()); @@ -66,13 +64,13 @@ internal static async Task ExecuteClick( await MainThreadSwitcher.SwitchToMainThread(ct); // Fire click events after expand animation so the user sees where the click lands - ExecutePointerClickEvents(resolvedTargets, pointerData); + pressTarget.Click(); MouseUiFrameWaitOutcome dissipateOutcome = await MouseUiOverlayAnimator.PlayDissipateAnimation(ct).ConfigureAwait(false); if (dissipateOutcome == MouseUiFrameWaitOutcome.TimedOut) { cleanupScheduler.QueueOverlayClear(); - return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, report); } if (dissipateOutcome == MouseUiFrameWaitOutcome.Paused) { @@ -83,7 +81,7 @@ internal static async Task ExecuteClick( } await MainThreadSwitcher.SwitchToMainThread(ct); - return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateClickResult(parameters, inputPos, report); } internal static async Task ExecuteLongPress( @@ -99,17 +97,16 @@ internal static async Task ExecuteLongPress( } Vector2 inputPos = new(parameters.X, parameters.Y); - Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); - PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); - MouseUiInputSystemSync.SyncMousePosition(screenPos); - ResolvedPointerTargets resolvedTargets = - MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, MouseAction.LongPress); - if (resolvedTargets.FailureResponse != null) + (IMouseUiPressTarget? resolvedTarget, SimulateMouseUiResponse? failure) = + ResolvePressTarget(parameters, eventSystem, inputPos, MouseAction.LongPress); + if (failure != null) { - return resolvedTargets.FailureResponse; + return failure; } - if (parameters.BypassRaycast && resolvedTargets.Target == null) + IMouseUiPressTarget pressTarget = resolvedTarget!; + MouseUiHitReport report = pressTarget.Report; + if (parameters.BypassRaycast && !report.HasHit) { return new SimulateMouseUiResponse { @@ -121,9 +118,7 @@ internal static async Task ExecuteLongPress( }; } - string? targetName = resolvedTargets.Target?.name; - bool hitTarget = resolvedTargets.Target != null; - bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null; + string? targetName = report.Name; SimulateMouseUiOverlayState.Update( MouseAction.LongPress, inputPos, null, Handles.GetMainGameViewSize()); @@ -143,7 +138,7 @@ internal static async Task ExecuteLongPress( } await MainThreadSwitcher.SwitchToMainThread(ct); - ExecuteLongPressPointerDown(resolvedTargets, pointerData); + pressTarget.PressDown(); try { @@ -157,11 +152,7 @@ internal static async Task ExecuteLongPress( finally { // Ensure pointerUp fires even if the hold loop is cancelled - if (shouldReleasePointer) - { - cleanupScheduler.ExecuteCleanupOnMainThread( - () => ExecuteEvents.Execute(resolvedTargets.Target!, pointerData, ExecuteEvents.pointerUpHandler)); - } + cleanupScheduler.ExecuteCleanupOnMainThread(pressTarget.Release); } // Why switch after the helper: HoldLongPressOrAbortAsync ends on the main thread, @@ -172,7 +163,7 @@ internal static async Task ExecuteLongPress( if (dissipateOutcome == MouseUiFrameWaitOutcome.TimedOut) { cleanupScheduler.QueueOverlayClear(); - return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, report); } if (dissipateOutcome == MouseUiFrameWaitOutcome.Paused) { @@ -183,7 +174,7 @@ internal static async Task ExecuteLongPress( } await MainThreadSwitcher.SwitchToMainThread(ct); - return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, targetName, hitTarget); + return MouseUiSimulationResponseFactory.CreateLongPressResult(parameters, inputPos, report); } private static SimulateMouseUiResponse? CreateInvalidLongPressDurationResponse(float duration) @@ -249,53 +240,23 @@ internal static async Task ExecuteLongPress( return null; } - private static void ExecutePointerClickEvents( - ResolvedPointerTargets resolvedTargets, - PointerEventData pointerData) - { - if (resolvedTargets.RawTarget == null) - { - return; - } - - if (resolvedTargets.PressTarget != null) - { - ExecuteEvents.ExecuteHierarchy( - resolvedTargets.RawTarget, - pointerData, - ExecuteEvents.pointerDownHandler); - } - - if (resolvedTargets.Target != null) - { - ExecuteEvents.Execute( - resolvedTargets.Target, - pointerData, - ExecuteEvents.pointerUpHandler); - } - - if (resolvedTargets.ClickTarget != null) - { - ExecuteEvents.Execute( - resolvedTargets.ClickTarget, - pointerData, - ExecuteEvents.pointerClickHandler); - } - } - - private static void ExecuteLongPressPointerDown( - ResolvedPointerTargets resolvedTargets, - PointerEventData pointerData) + private static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) ResolvePressTarget( + MouseUiSimulationCommand parameters, + EventSystem eventSystem, + Vector2 inputPos, + MouseAction action) { - if (resolvedTargets.RawTarget == null || resolvedTargets.Target == null) + Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); + PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); + MouseUiInputSystemSync.SyncMousePosition(screenPos); + ResolvedPointerTargets resolvedTargets = + MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, action); + if (resolvedTargets.FailureResponse != null) { - return; + return (null, resolvedTargets.FailureResponse); } - ExecuteEvents.ExecuteHierarchy( - resolvedTargets.RawTarget, - pointerData, - ExecuteEvents.pointerDownHandler); + return (new UguiPressTarget(resolvedTargets, pointerData), null); } } } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs index 32bf26260..0f8e5c920 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs @@ -66,13 +66,13 @@ internal static SimulateMouseUiResponse CreateFrameTimeoutResult( internal static SimulateMouseUiResponse CreateClickResult( MouseUiSimulationCommand parameters, Vector2 inputPos, - string? targetName, - bool hitTarget) + MouseUiHitReport hit) { + string? targetName = hit.Name; return new SimulateMouseUiResponse { Success = true, - Message = hitTarget + Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}'" : $"Clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1})" @@ -87,13 +87,13 @@ internal static SimulateMouseUiResponse CreateClickResult( internal static SimulateMouseUiResponse CreateLongPressResult( MouseUiSimulationCommand parameters, Vector2 inputPos, - string? targetName, - bool hitTarget) + MouseUiHitReport hit) { + string? targetName = hit.Name; return new SimulateMouseUiResponse { Success = true, - Message = hitTarget + Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}' for {parameters.Duration:F1}s" : $"Long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s" diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs new file mode 100644 index 000000000..629acfa36 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs @@ -0,0 +1,85 @@ +#nullable enable +using UnityEngine.EventSystems; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Delivers a click or long press to uGUI or physics targets through the EventSystem's handlers. With no + /// resolved target it reports no hit and sends nothing. + /// + internal sealed class UguiPressTarget : IMouseUiPressTarget + { + private readonly ResolvedPointerTargets _resolvedTargets; + private readonly PointerEventData? _pointerData; + private readonly bool _shouldReleasePointer; + + internal UguiPressTarget(ResolvedPointerTargets resolvedTargets, PointerEventData? pointerData) + { + _resolvedTargets = resolvedTargets; + _pointerData = pointerData; + + // Captured as plain values: Release can run from cleanup after a timeout, where checking the + // UnityEngine.Object targets for null is not safe. + bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null; + _shouldReleasePointer = shouldReleasePointer; + Report = new MouseUiHitReport(resolvedTargets.Target?.name, null, null); + } + + public MouseUiHitReport Report { get; } + + public void Click() + { + if (_resolvedTargets.RawTarget == null) + { + return; + } + + if (_resolvedTargets.PressTarget != null) + { + ExecuteEvents.ExecuteHierarchy( + _resolvedTargets.RawTarget, + _pointerData, + ExecuteEvents.pointerDownHandler); + } + + if (_resolvedTargets.Target != null) + { + ExecuteEvents.Execute( + _resolvedTargets.Target, + _pointerData, + ExecuteEvents.pointerUpHandler); + } + + if (_resolvedTargets.ClickTarget != null) + { + ExecuteEvents.Execute( + _resolvedTargets.ClickTarget, + _pointerData, + ExecuteEvents.pointerClickHandler); + } + } + + public void PressDown() + { + if (_resolvedTargets.RawTarget == null || _resolvedTargets.Target == null) + { + return; + } + + ExecuteEvents.ExecuteHierarchy( + _resolvedTargets.RawTarget, + _pointerData, + ExecuteEvents.pointerDownHandler); + } + + public void Release() + { + if (!_shouldReleasePointer) + { + return; + } + + ExecuteEvents.Execute(_resolvedTargets.Target!, _pointerData, ExecuteEvents.pointerUpHandler); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta new file mode 100644 index 000000000..8dd0ea9ee --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d99bca934334f489cb54b33b2dee12de +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef index 23bb2f8f5..4c9e61822 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor.asmdef @@ -6,6 +6,7 @@ "GUID:fc3fd32eddbee40e39c2d76dc184957b", "GUID:f1a38fcd7e85047c5a86331a3e37ddfc", "GUID:a2d87883023de4cf59b7d1962d5dd8aa", + "GUID:569bba870e5e64d0d94144057b927001", "GUID:aa7cf56cc5f074e57ba35272b877e116", "GUID:c956a21f824994ef087b6de566690b3d", "GUID:33fef506e6744f2982e8c13b1196f696", From fcc439cc9156cf3735760dcc0277ac33ba24c718 Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:46:18 +0900 Subject: [PATCH 4/8] feat: simulate-mouse-ui clicks and long presses reach UI Toolkit controls A click at a screenshot annotation's SimX/SimY on a UI Toolkit control used to report "no UI element hit": the uGUI bridge to UI Toolkit flips Y with Screen.height, which outside the player loop is the size of the last Editor view drawn, so the press landed beside the control. Clicks and long presses now look up the UI Toolkit control at the requested point directly and send it pointer events, so the result no longer depends on Screen. The uGUI side leaves out the bridge's panel hits. Where both are under the point, the uGUI hit wins only when its raycaster's sort priority is greater than the panel's sorting order, the same order the EventSystem applies to real input; a uGUI image without handlers in front still blocks the panel. UI Toolkit presses work without an EventSystem. A click with neither an EventSystem nor a UI Toolkit control under the point fails and names both. Bypass presses and drags still need an EventSystem and fail with the same message as before, and a bypass path containing '#' that matches nothing explains that UI Toolkit paths are not accepted there. Responses add UiSystem (UGUI, empty for physics targets, UIToolkit) and HitPath in the form annotations use, and UI Toolkit hits are named as such in the message. --- .../Tests/Editor/MouseUiHitPriorityTests.cs | 53 ++ .../Editor/MouseUiHitPriorityTests.cs.meta | 11 + .../MouseUiPointerTargetResolverTests.cs | 22 + .../MouseUiSimulationResponseFactoryTests.cs | 46 ++ .../Editor/UiRaycastHelperPanelSkipTests.cs | 108 +++ .../UiRaycastHelperPanelSkipTests.cs.meta | 11 + .../PlayMode/SimulateMouseUiToolkitTests.cs | 676 ++++++++++++++++++ .../SimulateMouseUiToolkitTests.cs.meta | 11 + .../Common/MouseUi/UiRaycastHelper.cs | 77 +- .../SimulateMouseUi/MouseUiHitPriority.cs | 21 + .../MouseUiHitPriority.cs.meta | 11 + .../SimulateMouseUi/MouseUiHitSystem.cs | 16 + .../SimulateMouseUi/MouseUiHitSystem.cs.meta | 11 + .../MouseUiPointerTargetResolver.cs | 4 +- .../MouseUiPressActionExecutor.cs | 16 +- .../MouseUiPressTargetResolver.cs | 102 +++ .../MouseUiPressTargetResolver.cs.meta | 11 + .../MouseUiSimulationResponseFactory.cs | 15 +- .../MouseUiSimulationValidator.cs | 12 +- .../SimulateMouseUiResponse.cs | 17 + .../SimulateMouseUi/SimulateMouseUiUseCase.cs | 22 +- .../SimulateMouseUi/UIToolkitPressTarget.cs | 42 ++ .../UIToolkitPressTarget.cs.meta | 11 + .../SimulateMouseUi/UguiPressTarget.cs | 18 +- 24 files changed, 1293 insertions(+), 51 deletions(-) create mode 100644 Assets/Tests/Editor/MouseUiHitPriorityTests.cs create mode 100644 Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta create mode 100644 Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs create mode 100644 Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta create mode 100644 Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs create mode 100644 Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs create mode 100644 Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta diff --git a/Assets/Tests/Editor/MouseUiHitPriorityTests.cs b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs new file mode 100644 index 000000000..3e4759214 --- /dev/null +++ b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs @@ -0,0 +1,53 @@ +using NUnit.Framework; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies which of an overlapping uGUI hit and UI Toolkit hit takes the input, following the EventSystem's + /// raycast ordering. + /// + public sealed class MouseUiHitPriorityTests + { + /// + /// Verifies a Canvas sorted above the panel puts uGUI in front. + /// + [Test] + public void IsUguiInFront_WhenCanvasSortsAbovePanel_ReturnsTrue() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(11, 10f), Is.True); + } + + /// + /// Verifies equal sort priorities leave UI Toolkit in front, since the panel's render order priority is + /// nearly int.MaxValue; a fractional panel sorting order is rounded down first. + /// + [TestCase(10, 10f)] + [TestCase(10, 10.5f)] + [TestCase(-1, -0.5f)] + public void IsUguiInFront_WhenSortPrioritiesAreEqual_ReturnsFalse(int uguiSortOrderPriority, float panelSortingOrder) + { + Assert.That(MouseUiHitPriority.IsUguiInFront(uguiSortOrderPriority, panelSortingOrder), Is.False); + } + + /// + /// Verifies a panel sorted above the Canvas stays in front. + /// + [Test] + public void IsUguiInFront_WhenPanelSortsAboveCanvas_ReturnsFalse() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(9, 10f), Is.False); + } + + /// + /// Verifies raycasters that keep the base priority, such as physics raycasters and non-overlay Canvases, + /// never come in front of a panel. + /// + [Test] + public void IsUguiInFront_WithBaseRaycasterPriority_ReturnsFalse() + { + Assert.That(MouseUiHitPriority.IsUguiInFront(int.MinValue, -1000f), Is.False); + } + } +} diff --git a/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta new file mode 100644 index 000000000..f1d7b7580 --- /dev/null +++ b/Assets/Tests/Editor/MouseUiHitPriorityTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 493c169e2fe5f485f956471f9840227a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs b/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs index d3281f00f..2f02091f9 100644 --- a/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs +++ b/Assets/Tests/Editor/MouseUiPointerTargetResolverTests.cs @@ -71,6 +71,28 @@ public void ResolveGameObjectPath_WithUniquePath_ReturnsTarget() Assert.That(failureResponse, Is.Null); } + /// + /// Verifies a GameObject whose name contains '#' still resolves by its path, so the UI Toolkit hint for + /// '#' paths only applies when nothing matches. + /// + [Test] + public void ResolveGameObjectPath_WithHashInGameObjectName_ReturnsTarget() + { + GameObject root = CreateGameObject("MouseUiTargetResolverTests_HashRoot"); + GameObject expectedTarget = CreateGameObject("Menu#Play", root.transform); + string targetPath = GameObjectPathUtility.GetFullPath(expectedTarget); + + (GameObject? target, SimulateMouseUiResponse? failureResponse) = + MouseUiPointerTargetResolver.ResolveGameObjectPath( + targetPath, + "TargetPath", + MouseAction.Click, + new Vector2(10f, 20f)); + + Assert.That(target, Is.SameAs(expectedTarget)); + Assert.That(failureResponse, Is.Null); + } + /// /// Verifies a missing hierarchy path returns the exact wire-visible failure response. /// diff --git a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs index a3a03654c..f09de75b6 100644 --- a/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs +++ b/Assets/Tests/Editor/MouseUiSimulationResponseFactoryTests.cs @@ -146,6 +146,52 @@ public void CreateLongPressResult_WithHitMode_MapsMessageAndFields( Assert.That(response.PositionY, Is.EqualTo(position.y)); } + /// + /// Verifies click and long-press results carry the hit's UI system and path, and name UI Toolkit hits as + /// such in the message. + /// + [Test] + public void CreatePressResults_WithUIToolkitHit_MapUiSystemAndPath() + { + MouseUiHitReport hit = new MouseUiHitReport("play", "Menu/Document#play", "UIToolkit"); + Vector2 position = new(200f, 125f); + + SimulateMouseUiResponse click = MouseUiSimulationResponseFactory.CreateClickResult( + CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.Click }), position, hit); + SimulateMouseUiResponse longPress = MouseUiSimulationResponseFactory.CreateLongPressResult( + CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.LongPress, Duration = 0.5f }), position, hit); + + Assert.That(click.Message, Is.EqualTo("Clicked UI Toolkit element 'play' at (200.0, 125.0)")); + Assert.That(longPress.Message, Is.EqualTo("Long-pressed UI Toolkit element 'play' at (200.0, 125.0) for 0.5s")); + foreach (SimulateMouseUiResponse response in new[] { click, longPress }) + { + Assert.That(response.HitGameObjectName, Is.EqualTo("play")); + Assert.That(response.UiSystem, Is.EqualTo("UIToolkit")); + Assert.That(response.HitPath, Is.EqualTo("Menu/Document#play")); + } + } + + /// + /// Verifies a uGUI hit keeps its message and carries its UI system and path, and no hit leaves both null. + /// + [Test] + public void CreateClickResult_WithUguiHitOrNoHit_MapsUiSystemAndPath() + { + MouseUiSimulationCommand command = CreateCommand(new SimulateMouseUiSchema { Action = UnityCliLoopMouseUiAction.Click }); + Vector2 position = new(10f, 20f); + + SimulateMouseUiResponse ugui = MouseUiSimulationResponseFactory.CreateClickResult( + command, position, new MouseUiHitReport("Ok", "Canvas/Ok", "UGUI")); + SimulateMouseUiResponse none = MouseUiSimulationResponseFactory.CreateClickResult( + command, position, MouseUiHitReport.None); + + Assert.That(ugui.Message, Is.EqualTo("Clicked 'Ok' at (10.0, 20.0)")); + Assert.That(ugui.UiSystem, Is.EqualTo("UGUI")); + Assert.That(ugui.HitPath, Is.EqualTo("Canvas/Ok")); + Assert.That(none.UiSystem, Is.Null); + Assert.That(none.HitPath, Is.Null); + } + /// /// Verifies one-shot drag results retain normal and bypass message contracts. /// diff --git a/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs new file mode 100644 index 000000000..aa6714045 --- /dev/null +++ b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using NUnit.Framework; +using UnityEngine; +using UnityEngine.EventSystems; +using UnityEngine.UI; +using UnityEngine.UIElements; + +using io.github.hatayama.UnityCliLoop.FirstPartyTools; + +namespace io.github.hatayama.UnityCliLoop.Tests.Editor +{ + /// + /// Verifies how EventSystem results that come from uGUI's UI Toolkit bridge are handled when choosing a hit. + /// The results are built by hand with real raycaster components as their module, because in EditMode the + /// bridge raycaster never registers and returns nothing. No Canvas-space hit is passed, since comparing one + /// reads Screen. + /// + public sealed class UiRaycastHelperPanelSkipTests + { + private readonly List _createdObjects = new List(); + + [TearDown] + public void TearDown() + { + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.DestroyImmediate(createdObject); + } + } + + _createdObjects.Clear(); + } + + /// + /// Verifies a bridge hit in first place is skipped and the next hit is chosen. + /// + [Test] + public void ChooseHitSkippingUIToolkitPanels_WhenFirstHitIsPanelRaycaster_ReturnsNextHit() + { + RaycastResult panelHit = CreatePanelHit(); + RaycastResult graphicHit = CreateGraphicHit(); + + RaycastResult? chosen = UiRaycastHelper.ChooseHitSkippingUIToolkitPanels( + new List { panelHit, graphicHit }, null); + + Assert.That(chosen.HasValue, Is.True); + Assert.That(chosen.Value.gameObject, Is.SameAs(graphicHit.gameObject)); + } + + /// + /// Verifies only bridge hits leave nothing to choose. + /// + [Test] + public void ChooseHitSkippingUIToolkitPanels_WhenOnlyPanelRaycaster_ReturnsNull() + { + RaycastResult? chosen = UiRaycastHelper.ChooseHitSkippingUIToolkitPanels( + new List { CreatePanelHit() }, null); + + Assert.That(chosen.HasValue, Is.False); + } + + /// + /// Verifies the unfiltered choice still returns a bridge hit in first place, so other callers keep their + /// current results. + /// + [Test] + public void ChooseHit_WhenFirstHitIsPanelRaycaster_ReturnsIt() + { + RaycastResult panelHit = CreatePanelHit(); + + RaycastResult? chosen = UiRaycastHelper.ChooseHit( + new List { panelHit, CreateGraphicHit() }, null); + + Assert.That(chosen.HasValue, Is.True); + Assert.That(chosen.Value.gameObject, Is.SameAs(panelHit.gameObject)); + } + + /// + /// Verifies the type name the bridge results are recognised by matches the uGUI type. + /// + [Test] + public void PanelRaycasterTypeFullName_MatchesUguiType() + { + Assert.That(UiRaycastHelper.PanelRaycasterTypeFullName, Is.EqualTo(typeof(PanelRaycaster).FullName)); + } + + private RaycastResult CreatePanelHit() + { + GameObject panelObject = CreateGameObject("Panel"); + return new RaycastResult { gameObject = panelObject, module = panelObject.AddComponent() }; + } + + private RaycastResult CreateGraphicHit() + { + GameObject canvasObject = CreateGameObject("Canvas"); + return new RaycastResult { gameObject = canvasObject, module = canvasObject.AddComponent() }; + } + + private GameObject CreateGameObject(string name) + { + GameObject gameObject = new GameObject("UiRaycastHelperPanelSkipTests_" + name); + _createdObjects.Add(gameObject); + return gameObject; + } + } +} diff --git a/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta new file mode 100644 index 000000000..20cc20769 --- /dev/null +++ b/Assets/Tests/Editor/UiRaycastHelperPanelSkipTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2653f3ed2aaa646dc9186abf235ba686 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs new file mode 100644 index 000000000..b2c22adb2 --- /dev/null +++ b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs @@ -0,0 +1,676 @@ +#nullable enable +using System.Collections; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using io.github.hatayama.UnityCliLoop.FirstPartyTools; +using io.github.hatayama.UnityCliLoop.Runtime; +using io.github.hatayama.UnityCliLoop.ToolContracts; +using Newtonsoft.Json.Linq; +using NUnit.Framework; +using UnityEditor; +using UnityEngine; +using UnityEngine.EventSystems; +using UnityEngine.TestTools; +using UnityEngine.UI; +using UnityEngine.UIElements; + +using Image = UnityEngine.UI.Image; +using MouseButton = io.github.hatayama.UnityCliLoop.Runtime.MouseButton; +using UIToolkitButton = UnityEngine.UIElements.Button; + +namespace io.github.hatayama.UnityCliLoop.Tests.PlayMode +{ + /// + /// Verifies simulate-mouse-ui clicks and long presses reach UI Toolkit controls, choose between uGUI and UI + /// Toolkit where both are under the point as the EventSystem would, work without an EventSystem for UI + /// Toolkit, and report which UI system and path were hit. The UI Toolkit button sits at the centre of the + /// Game View, where the Canvas elements are anchored too. Each test decides whether an EventSystem exists. + /// + public sealed class SimulateMouseUiToolkitTests + { + private const string RuntimeThemePath = "Assets/UI Toolkit/UnityThemes/UnityDefaultRuntimeTheme.tss"; + private const string DocumentName = "SimulateMouseUiToolkitTests_Document"; + private const float ButtonWidth = 200f; + private const float ButtonHeight = 100f; + private const string NoEventSystemMessage = "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."; + + private readonly List _createdObjects = new List(); + private ExistingEventSystemDisableScope _eventSystemDisableScope = null!; + private GameObject _canvasObject = null!; + private SimulateMouseUiTool _tool = null!; + private SimulateMouseUiResponse _lastResponse = null!; + + [UnitySetUp] + public IEnumerator SetUp() + { + _eventSystemDisableScope = new ExistingEventSystemDisableScope(); + + _canvasObject = new GameObject("TestCanvas"); + _createdObjects.Add(_canvasObject); + Canvas canvas = _canvasObject.AddComponent(); + canvas.renderMode = RenderMode.ScreenSpaceOverlay; + _canvasObject.AddComponent(); + _canvasObject.AddComponent(); + + _tool = new SimulateMouseUiTool(); + yield return null; + } + + [UnityTearDown] + public IEnumerator TearDown() + { + MouseDragState.Clear(); + foreach (Object createdObject in _createdObjects) + { + if (createdObject != null) + { + Object.Destroy(createdObject); + } + } + + _createdObjects.Clear(); + yield return null; + + _eventSystemDisableScope.Restore(); + } + + /// + /// Verifies a uGUI click reports the UGUI system and the target's hierarchy path (#2). + /// + [UnityTest] + public IEnumerator Click_OnUgui_ReportsUguiSystemAndPath() + { + CreateEventSystem(); + ClickTracker tracker = CreateCanvasClickTarget("UguiTarget"); + yield return null; + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("UguiTarget", _lastResponse.HitGameObjectName); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + Assert.AreEqual("TestCanvas/UguiTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a click on a UI Toolkit button clicks it once, reports it, and leaves no pointer capture (#4). + /// + [UnityTest] + public IEnumerator Click_OnUIToolkitButtonWithEventSystem_ClicksIt() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("play", _lastResponse.HitGameObjectName); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + Assert.AreEqual(DocumentName + "#play", _lastResponse.HitPath); + Assert.AreEqual( + $"Clicked UI Toolkit element 'play' at ({GameViewCenter().x:F1}, {GameViewCenter().y:F1})", + _lastResponse.Message); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies a Canvas sorted above the panel takes the click (#5). + /// + [UnityTest] + public IEnumerator Click_WhereUguiCanvasSortsAbovePanel_ClicksUgui() + { + yield return ClickOverlap(canvasSortingOrder: 11, panelSortingOrder: 10f); + + Assert.IsTrue(_lastTracker.PointerClickCalled, "the Canvas element must be clicked"); + Assert.AreEqual(0, _lastCounter.Clicks, "the UI Toolkit button must not be clicked"); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + } + + /// + /// Verifies a Canvas and a panel with the same sorting order leave UI Toolkit in front (#6). + /// + [UnityTest] + public IEnumerator Click_WhereCanvasAndPanelShareSortingOrder_ClicksUIToolkit() + { + yield return ClickOverlap(canvasSortingOrder: 10, panelSortingOrder: 10f); + + Assert.IsFalse(_lastTracker.PointerClickCalled, "the Canvas element must not be clicked"); + Assert.AreEqual(1, _lastCounter.Clicks, "the UI Toolkit button must be clicked"); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a panel sorted above the Canvas takes the click (#7). + /// + [UnityTest] + public IEnumerator Click_WherePanelSortsAboveCanvas_ClicksUIToolkit() + { + yield return ClickOverlap(canvasSortingOrder: 9, panelSortingOrder: 10f); + + Assert.IsFalse(_lastTracker.PointerClickCalled, "the Canvas element must not be clicked"); + Assert.AreEqual(1, _lastCounter.Clicks, "the UI Toolkit button must be clicked"); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a UI Toolkit button is clicked when the scene has no EventSystem (#9). + /// + [UnityTest] + public IEnumerator Click_OnUIToolkitButtonWithoutEventSystem_ClicksIt() + { + Assert.IsNull(EventSystem.current, "Setup: no EventSystem must be active."); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + } + + /// + /// Verifies a click with no EventSystem and no UI Toolkit element under the point fails and names both + /// UI systems (#10). + /// + [UnityTest] + public IEnumerator Click_WithoutEventSystemOnEmptyPoint_FailsNamingBothSystems() + { + Vector2 point = GameViewCenter(); + + yield return RunClick(point); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual( + $"No EventSystem found in the scene, and no UI Toolkit element is at ({point.x:F1}, {point.y:F1}). Ensure an EventSystem GameObject exists for uGUI.", + _lastResponse.Message); + } + + /// + /// Verifies a bypass click given a UI Toolkit path explains that such paths are not accepted (#15). + /// + [UnityTest] + public IEnumerator BypassClick_WithUIToolkitPath_ExplainsPathIsNotAccepted() + { + CreateEventSystem(); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + string path = DocumentName + "#play"; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = path + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual( + $"TargetPath '{path}' was not found. UI Toolkit element paths are not accepted here; click the element at its SimX/SimY without --bypass-raycast.", + _lastResponse.Message); + Assert.AreEqual(0, counter.Clicks); + } + + /// + /// Verifies a bypass click still needs an EventSystem and fails with the same message as before (#17). + /// + [UnityTest] + public IEnumerator BypassClick_WithoutEventSystem_FailsAsBefore() + { + CreateCanvasClickTarget("BypassTarget"); + yield return null; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = "TestCanvas/BypassTarget" + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual(NoEventSystemMessage, _lastResponse.Message); + } + + /// + /// Verifies option errors are reported before a missing EventSystem (#18). + /// + [UnityTest] + public IEnumerator Drag_WithNegativeDragSpeedAndNoEventSystem_ReportsDragSpeedError() + { + yield return RunTool(new JObject + { + ["action"] = MouseAction.Drag.ToString(), + ["dragSpeed"] = -1f + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual("DragSpeed must be non-negative, got: -1", _lastResponse.Message); + } + + /// + /// Verifies a drag with no EventSystem still fails with the existing message. + /// + [UnityTest] + public IEnumerator Drag_WithoutEventSystem_FailsAsBefore() + { + CreateDocumentWithCenteredButton(0f, out ClickCounter _); + yield return WaitForLayout(); + Vector2 point = GameViewCenter(); + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Drag.ToString(), + ["fromX"] = point.x, + ["fromY"] = point.y, + ["x"] = point.x + 10f, + ["y"] = point.y + }); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual(NoEventSystemMessage, _lastResponse.Message); + } + + /// + /// Verifies a right click reaches the UI Toolkit button as button 1 and does not click it (#20). + /// + [UnityTest] + public IEnumerator RightClick_OnUIToolkitButton_DoesNotClick() + { + CreateEventSystem(); + CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["x"] = GameViewCenter().x, + ["y"] = GameViewCenter().y, + ["button"] = MouseButton.Right.ToString() + }); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.LastPressedButton, "the press must arrive as the right button"); + Assert.AreEqual(0, counter.Clicks); + } + + /// + /// Verifies a click during a uGUI split drag is rejected as drag-active even after the EventSystem is + /// removed (#23). + /// + [UnityTest] + public IEnumerator Click_WhileSplitDragActiveWithoutEventSystem_IsRejectedAsDragActive() + { + GameObject eventSystemObject = CreateEventSystem(); + GameObject dragObject = CreateCanvasElement("DragTarget"); + dragObject.AddComponent(); + dragObject.AddComponent(); + yield return null; + Vector2 point = GameViewCenter(); + yield return RunTool(new JObject + { + ["action"] = MouseAction.DragStart.ToString(), + ["x"] = point.x, + ["y"] = point.y + }); + Assert.IsTrue(_lastResponse.Success, "Setup: DragStart must succeed. " + _lastResponse.Message); + eventSystemObject.SetActive(false); + yield return null; + + yield return RunClick(point); + + Assert.IsFalse(_lastResponse.Success); + Assert.AreEqual("Cannot Click while a split drag is active. Call DragEnd first.", _lastResponse.Message); + } + + /// + /// Verifies a handler-less uGUI image sorted above the panel stops the click from reaching UI Toolkit, + /// as it would for real input (#24). + /// + [UnityTest] + public IEnumerator Click_WhereHandlerlessUguiImageSortsAbovePanel_BlocksUIToolkit() + { + CreateEventSystem(); + _canvasObject.GetComponent().sortingOrder = 11; + CreateCanvasElement("Blocker").AddComponent(); + CreateDocumentWithCenteredButton(10f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(0, counter.Clicks, "the UI Toolkit button must not be clicked"); + Assert.IsNull(_lastResponse.HitGameObjectName); + Assert.IsNull(_lastResponse.UiSystem); + Assert.IsNull(_lastResponse.HitPath); + } + + /// + /// Verifies a click still succeeds without error when the UI Toolkit document is disabled after the + /// target is resolved and before the press is sent, and nothing is clicked (#25). + /// + [UnityTest] + public IEnumerator Click_WhenUIToolkitDocumentDisabledBeforeDispatch_SkipsWithoutError() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + Task task = StartTool(ClickParameters(GameViewCenter())); + + yield return null; + Assert.IsFalse(task.IsCompleted, "Setup: the click must still be animating when the document is disabled."); + document.enabled = false; + yield return WaitForTask(task); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(0, counter.Clicks); + Assert.AreEqual("play", _lastResponse.HitGameObjectName); + } + + /// + /// Verifies a Canvas image with no pointer handler reports no hit (#26). + /// + [UnityTest] + public IEnumerator Click_OnUguiImageWithoutHandler_ReportsNoHit() + { + CreateEventSystem(); + CreateCanvasElement("PlainImage").AddComponent(); + yield return null; + Vector2 point = GameViewCenter(); + + yield return RunClick(point); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual($"Clicked at ({point.x:F1}, {point.y:F1}) - no UI element hit", _lastResponse.Message); + Assert.IsNull(_lastResponse.HitGameObjectName); + Assert.IsNull(_lastResponse.UiSystem); + Assert.IsNull(_lastResponse.HitPath); + } + + /// + /// Verifies a click on a physics target outside any Canvas reports an empty UI system and its path (#27). + /// + [UnityTest] + public IEnumerator Click_OnPhysics2DTarget_ReportsEmptyUiSystem() + { + CreateEventSystem(); + GameObject physicsRoot = new GameObject("Physics2DRaycasterRoot"); + _createdObjects.Add(physicsRoot); + GameObject cameraObject = new GameObject("Physics2DCamera"); + cameraObject.transform.SetParent(physicsRoot.transform, false); + cameraObject.transform.position = new Vector3(0f, 0f, -10f); + Camera physicsCamera = cameraObject.AddComponent(); + physicsCamera.orthographic = true; + physicsCamera.orthographicSize = 5f; + cameraObject.AddComponent(); + GameObject physicsTarget = new GameObject("Physics2DTarget"); + physicsTarget.transform.SetParent(physicsRoot.transform, false); + Vector3 screenPoint = new Vector3(Screen.width / 2f, Screen.height / 2f, 10f); + Vector3 worldPoint = physicsCamera.ScreenToWorldPoint(screenPoint); + physicsTarget.transform.position = new Vector3(worldPoint.x, worldPoint.y, 0f); + physicsTarget.AddComponent().size = new Vector2(2f, 2f); + ClickTracker tracker = physicsTarget.AddComponent(); + yield return null; + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("Physics2DTarget", _lastResponse.HitGameObjectName); + Assert.AreEqual("", _lastResponse.UiSystem); + Assert.AreEqual("Physics2DRaycasterRoot/Physics2DTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a bypass click on a Canvas element reports the UGUI system and the target path (#28). + /// + [UnityTest] + public IEnumerator BypassClick_OnCanvasButton_ReportsUguiSystem() + { + CreateEventSystem(); + ClickTracker tracker = CreateCanvasClickTarget("BypassTarget"); + yield return null; + + yield return RunTool(new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["bypassRaycast"] = true, + ["targetPath"] = "TestCanvas/BypassTarget" + }); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.IsTrue(tracker.PointerClickCalled); + Assert.AreEqual("UGUI", _lastResponse.UiSystem); + Assert.AreEqual("TestCanvas/BypassTarget", _lastResponse.HitPath); + } + + /// + /// Verifies a long press on a UI Toolkit button clicks it once on release, as UI Toolkit buttons do, and + /// leaves no pointer capture. + /// + [UnityTest] + public IEnumerator LongPress_OnUIToolkitButton_ClicksOnReleaseAndLeavesNoCapture() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + + yield return RunTool(LongPressParameters(GameViewCenter(), 0.2f)); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual(1, counter.Clicks); + Assert.AreEqual("UIToolkit", _lastResponse.UiSystem); + Assert.AreEqual(DocumentName + "#play", _lastResponse.HitPath); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies cancelling a long press while it holds still releases the UI Toolkit press, so no element is + /// left holding the pointer. + /// + [UnityTest] + public IEnumerator LongPress_OnUIToolkitButton_WhenCancelledDuringHold_ReleasesCapture() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + using CancellationTokenSource cancellation = new CancellationTokenSource(); + Task task = _tool.ExecuteAsync(LongPressParameters(GameViewCenter(), 5f), cancellation.Token); + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => counter.PointerDowns > 0 || Time.realtimeSinceStartup >= timeoutAt); + Assert.AreEqual(1, counter.PointerDowns, "Setup: the press must have arrived before cancelling."); + + cancellation.Cancel(); + timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => task.IsCompleted || Time.realtimeSinceStartup >= timeoutAt); + yield return null; + + Assert.IsTrue(task.IsCompleted, "the long press must end after cancelling"); + Assert.IsTrue(task.IsCanceled || task.Exception?.InnerException is System.OperationCanceledException, + $"the long press must end by cancellation, got {task.Status}: {task.Exception}"); + Assert.IsNull(document.rootVisualElement.panel.GetCapturingElement(PointerId.mousePointerId)); + } + + /// + /// Verifies a long press whose UI Toolkit document is disabled while it holds ends successfully without + /// error. + /// + [UnityTest] + public IEnumerator LongPress_WhenDocumentDisabledDuringHold_ReleasesWithoutError() + { + CreateEventSystem(); + UIDocument document = CreateDocumentWithCenteredButton(0f, out ClickCounter counter); + yield return WaitForLayout(); + Task task = StartTool(LongPressParameters(GameViewCenter(), 0.5f)); + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => counter.PointerDowns > 0 || Time.realtimeSinceStartup >= timeoutAt); + Assert.AreEqual(1, counter.PointerDowns, "Setup: the press must have arrived before disabling."); + + document.enabled = false; + yield return WaitForTask(task); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + Assert.AreEqual("play", _lastResponse.HitGameObjectName); + } + + private ClickTracker _lastTracker = null!; + private ClickCounter _lastCounter = null!; + + private IEnumerator ClickOverlap(int canvasSortingOrder, float panelSortingOrder) + { + CreateEventSystem(); + _canvasObject.GetComponent().sortingOrder = canvasSortingOrder; + _lastTracker = CreateCanvasClickTarget("UguiTarget"); + CreateDocumentWithCenteredButton(panelSortingOrder, out _lastCounter); + yield return WaitForLayout(); + + yield return RunClick(GameViewCenter()); + + Assert.IsTrue(_lastResponse.Success, _lastResponse.Message); + } + + private GameObject CreateEventSystem() + { + GameObject eventSystemObject = new GameObject("TestEventSystem"); + _createdObjects.Add(eventSystemObject); + eventSystemObject.AddComponent(); + eventSystemObject.AddComponent(); + return eventSystemObject; + } + + private ClickTracker CreateCanvasClickTarget(string name) + { + GameObject element = CreateCanvasElement(name); + element.AddComponent(); + return element.AddComponent(); + } + + // Anchored at the Canvas centre, which is the Game View centre in PlayMode. + private GameObject CreateCanvasElement(string name) + { + GameObject element = new GameObject(name); + element.transform.SetParent(_canvasObject.transform, false); + RectTransform rect = element.AddComponent(); + rect.anchorMin = new Vector2(0.5f, 0.5f); + rect.anchorMax = new Vector2(0.5f, 0.5f); + rect.anchoredPosition = Vector2.zero; + rect.sizeDelta = new Vector2(ButtonWidth, ButtonHeight); + return element; + } + + // The object starts inactive so the document sees its panel settings when it first enables. + private UIDocument CreateDocumentWithCenteredButton(float sortingOrder, out ClickCounter counter) + { + PanelSettings settings = ScriptableObject.CreateInstance(); + settings.scaleMode = PanelScaleMode.ConstantPixelSize; + settings.scale = 1f; + settings.sortingOrder = sortingOrder; + settings.themeStyleSheet = AssetDatabase.LoadAssetAtPath(RuntimeThemePath); + Assert.IsNotNull(settings.themeStyleSheet, "the runtime theme must exist at " + RuntimeThemePath); + _createdObjects.Add(settings); + + GameObject documentObject = new GameObject(DocumentName); + documentObject.SetActive(false); + _createdObjects.Add(documentObject); + UIDocument document = documentObject.AddComponent(); + document.panelSettings = settings; + documentObject.SetActive(true); + + UIToolkitButton button = new UIToolkitButton { name = "play" }; + Vector2 center = GameViewCenter(); + button.style.position = Position.Absolute; + button.style.left = center.x - ButtonWidth / 2f; + button.style.top = center.y - ButtonHeight / 2f; + button.style.width = ButtonWidth; + button.style.height = ButtonHeight; + button.style.marginLeft = 0f; + button.style.marginTop = 0f; + counter = new ClickCounter(button); + document.rootVisualElement.Add(button); + return document; + } + + private static Vector2 GameViewCenter() + { + Vector2 size = Handles.GetMainGameViewSize(); + return new Vector2(Mathf.Round(size.x / 2f), Mathf.Round(size.y / 2f)); + } + + private static JObject ClickParameters(Vector2 point) + { + return new JObject + { + ["action"] = MouseAction.Click.ToString(), + ["x"] = point.x, + ["y"] = point.y + }; + } + + private static JObject LongPressParameters(Vector2 point, float duration) + { + return new JObject + { + ["action"] = MouseAction.LongPress.ToString(), + ["x"] = point.x, + ["y"] = point.y, + ["duration"] = duration + }; + } + + private IEnumerator RunClick(Vector2 point) + { + yield return RunTool(ClickParameters(point)); + } + + private IEnumerator RunTool(JObject parameters) + { + yield return WaitForTask(StartTool(parameters)); + } + + private Task StartTool(JObject parameters) + { + return _tool.ExecuteAsync(parameters, CancellationToken.None); + } + + private IEnumerator WaitForTask(Task task) + { + float timeoutAt = Time.realtimeSinceStartup + 5f; + yield return new WaitUntil(() => task.IsCompleted || Time.realtimeSinceStartup >= timeoutAt); + Assert.IsTrue(task.IsCompleted, "Tool execution timed out."); + Assert.AreEqual(TaskStatus.RanToCompletion, task.Status, $"Tool execution should complete: {task.Exception}"); + _lastResponse = (SimulateMouseUiResponse)task.Result; + } + + // The root's size is set by the runtime panel update in the player loop, so frames must pass. + private static IEnumerator WaitForLayout() + { + yield return null; + yield return null; + } + + /// + /// Counts clicks and presses on a UI Toolkit button. + /// + private sealed class ClickCounter + { + internal ClickCounter(UIToolkitButton button) + { + button.clicked += () => Clicks++; + button.RegisterCallback( + evt => + { + PointerDowns++; + LastPressedButton = evt.button; + }, + TrickleDown.TrickleDown); + } + + internal int Clicks { get; private set; } + internal int PointerDowns { get; private set; } + internal int LastPressedButton { get; private set; } = -1; + } + } +} diff --git a/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta new file mode 100644 index 000000000..c6d8b4763 --- /dev/null +++ b/Assets/Tests/PlayMode/SimulateMouseUiToolkitTests.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: cbbacf9a9c5134f308de10793737999e +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs index 356af9dc1..b6774147d 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/UiRaycastHelper.cs @@ -12,12 +12,58 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// internal static class UiRaycastHelper { + // Results from uGUI's bridge to UI Toolkit panels are recognised by type name because the type is + // compiled only when the UI Toolkit package is present. They are skipped where a caller resolves UI + // Toolkit itself: the bridge flips Y with Screen.height, which outside the player loop is the size of + // the last Editor view drawn, so its hits land at the wrong place on the panel. + internal const string PanelRaycasterTypeFullName = "UnityEngine.UIElements.PanelRaycaster"; + public static RaycastResult? RaycastUI(Vector2 screenPosition, EventSystem eventSystem) { RaycastContext context = new(eventSystem); return context.Raycast(screenPosition); } + internal static RaycastResult? RaycastUISkippingUIToolkitPanels(Vector2 screenPosition, EventSystem eventSystem) + { + RaycastContext context = new(eventSystem); + return context.RaycastSkippingUIToolkitPanels(screenPosition); + } + + // Picks the EventSystem's first hit, unless the Canvas-space hit should win over it; with no + // EventSystem hit, the Canvas-space hit. + internal static RaycastResult? ChooseHit(List eventSystemResults, RaycastResult? canvasSpaceHit) + { + if (eventSystemResults.Count > 0) + { + RaycastResult firstHit = eventSystemResults[0]; + + if (canvasSpaceHit != null && ShouldPreferCanvasSpaceHit(canvasSpaceHit.Value, firstHit)) + { + return canvasSpaceHit; + } + + return firstHit; + } + + // EventSystem clips at Screen.width/height, which can be smaller than the + // Canvas layout space (Game view target resolution). Fall back to manual hit testing. + return canvasSpaceHit; + } + + internal static RaycastResult? ChooseHitSkippingUIToolkitPanels( + List eventSystemResults, + RaycastResult? canvasSpaceHit) + { + List withoutPanels = eventSystemResults.FindAll(result => !IsPanelRaycasterHit(result)); + return ChooseHit(withoutPanels, canvasSpaceHit); + } + + private static bool IsPanelRaycasterHit(RaycastResult result) + { + return result.module != null && result.module.GetType().FullName == PanelRaycasterTypeFullName; + } + private static List CollectCanvasRaycastSources() { #if UNITY_6000_4_OR_NEWER @@ -106,27 +152,24 @@ internal RaycastContext(EventSystem eventSystem) } public RaycastResult? Raycast(Vector2 screenPosition) + { + RaycastResult? canvasSpaceHit = RaycastBothWays(screenPosition); + return ChooseHit(_eventSystemResults, canvasSpaceHit); + } + + internal RaycastResult? RaycastSkippingUIToolkitPanels(Vector2 screenPosition) + { + RaycastResult? canvasSpaceHit = RaycastBothWays(screenPosition); + return ChooseHitSkippingUIToolkitPanels(_eventSystemResults, canvasSpaceHit); + } + + // Fills the EventSystem results and returns the Canvas-space hit for the same point. + private RaycastResult? RaycastBothWays(Vector2 screenPosition) { _eventSystemResults.Clear(); _pointerData.position = screenPosition; _eventSystem.RaycastAll(_pointerData, _eventSystemResults); - RaycastResult? canvasSpaceHit = RaycastCanvasSpaceFromSources(screenPosition, _canvasRaycastSources); - - if (_eventSystemResults.Count > 0) - { - RaycastResult firstHit = _eventSystemResults[0]; - - if (canvasSpaceHit != null && ShouldPreferCanvasSpaceHit(canvasSpaceHit.Value, firstHit)) - { - return canvasSpaceHit; - } - - return firstHit; - } - - // EventSystem clips at Screen.width/height, which can be smaller than the - // Canvas layout space (Game view target resolution). Fall back to manual hit testing. - return canvasSpaceHit; + return RaycastCanvasSpaceFromSources(screenPosition, _canvasRaycastSources); } } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs new file mode 100644 index 000000000..acdcb32cf --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs @@ -0,0 +1,21 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides whether a uGUI hit or a UI Toolkit hit at the same point takes the input, the way the + /// EventSystem orders them for real input. + /// + internal static class MouseUiHitPriority + { + // The EventSystem compares hits from different raycasters by camera depth only when both have a camera, + // and the UI Toolkit raycaster has none; then by sortOrderPriority, where a panel reports its sorting + // order rounded down; then by renderOrderPriority, where a panel reports nearly int.MaxValue. So uGUI is + // in front only when its sortOrderPriority is strictly greater. + internal static bool IsUguiInFront(int uguiSortOrderPriority, float panelSortingOrder) + { + return uguiSortOrderPriority > Mathf.FloorToInt(panelSortingOrder); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta new file mode 100644 index 000000000..3f23b4481 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitPriority.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9f8021d3f2b704b10a0a03913c7c54bf +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs new file mode 100644 index 000000000..d749af200 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs @@ -0,0 +1,16 @@ +#nullable enable + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// The UiSystem values simulate-mouse-ui reports, matching the ones screenshot annotations use. + /// + internal static class MouseUiHitSystem + { + internal const string UGUI = "UGUI"; + internal const string UIToolkit = "UIToolkit"; + + // A target outside any Canvas, such as an object hit by a physics raycaster. + internal const string NonUi = ""; + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta new file mode 100644 index 000000000..099173acd --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiHitSystem.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 6bf90b44c9012482a8ddbd660976fd13 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs index 2dedc3bd7..26b918ea0 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPointerTargetResolver.cs @@ -51,7 +51,9 @@ internal static ResolvedPointerTargets ResolvePressablePointerTargets( return ResolveBypassPressablePointerTargets(parameters, inputPos, pointerData, action); } - RaycastResult? hit = UiRaycastHelper.RaycastUI(screenPos, eventSystem); + // UI Toolkit is resolved separately at the requested point; the bridge's panel hits are flipped with + // Screen.height and would land on the wrong element. + RaycastResult? hit = UiRaycastHelper.RaycastUISkippingUIToolkitPanels(screenPos, eventSystem); if (hit == null) { return ResolvedPointerTargets.Empty; diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs index 7f0755d77..3244e05fa 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressActionExecutor.cs @@ -17,7 +17,7 @@ internal static class MouseUiPressActionExecutor { internal static async Task ExecuteClick( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, MouseUiMainThreadCleanupScheduler cleanupScheduler, CancellationToken ct) { @@ -86,7 +86,7 @@ internal static async Task ExecuteClick( internal static async Task ExecuteLongPress( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, MouseUiMainThreadCleanupScheduler cleanupScheduler, CancellationToken ct) { @@ -242,21 +242,13 @@ internal static async Task ExecuteLongPress( private static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) ResolvePressTarget( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, Vector2 inputPos, MouseAction action) { Vector2 screenPos = MouseUiCoordinateConverter.InputToScreen(inputPos); - PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); MouseUiInputSystemSync.SyncMousePosition(screenPos); - ResolvedPointerTargets resolvedTargets = - MouseUiPointerTargetResolver.ResolvePressablePointerTargets(parameters, eventSystem, inputPos, screenPos, pointerData, action); - if (resolvedTargets.FailureResponse != null) - { - return (null, resolvedTargets.FailureResponse); - } - - return (new UguiPressTarget(resolvedTargets, pointerData), null); + return MouseUiPressTargetResolver.Resolve(parameters, eventSystem, inputPos, screenPos, action); } } } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs new file mode 100644 index 000000000..718bab3b9 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs @@ -0,0 +1,102 @@ +#nullable enable +using UnityEngine; +using UnityEngine.EventSystems; + +using io.github.hatayama.UnityCliLoop.Runtime; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Decides whether a click or long press goes to uGUI or to UI Toolkit. On success a target is always + /// returned, an empty uGUI one when nothing is under the point; on failure only the response is. + /// + internal static class MouseUiPressTargetResolver + { + private const string NoEventSystemMessage = "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."; + private const string UIToolkitPathHint = + " UI Toolkit element paths are not accepted here; click the element at its SimX/SimY without --bypass-raycast."; + + internal static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) Resolve( + MouseUiSimulationCommand parameters, + EventSystem? eventSystem, + Vector2 inputPos, + Vector2 screenPos, + MouseAction action) + { + if (parameters.BypassRaycast) + { + return ResolveBypass(parameters, eventSystem, inputPos, screenPos, action); + } + + // UI Toolkit is looked up at the requested point itself, which does not depend on Screen; the uGUI + // side leaves out the bridge's panel hits, which do. + UIToolkitPointerTarget? toolkit = UIToolkitScreenElements.FindPointerTarget(inputPos); + ResolvedPointerTargets uguiResolved = ResolvedPointerTargets.Empty; + PointerEventData? uguiPointerData = null; + if (eventSystem != null) + { + uguiPointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); + uguiResolved = MouseUiPointerTargetResolver.ResolvePressablePointerTargets( + parameters, eventSystem, inputPos, screenPos, uguiPointerData, action); + } + + // A raycast hit without a handler still counts: real input stops at it and never reaches the panel. + bool uguiHasRaycastHit = uguiResolved.RawTarget != null; + if (toolkit == null && !uguiHasRaycastHit) + { + if (eventSystem == null) + { + return (null, MouseUiSimulationResponseFactory.CreateFailure( + parameters, + $"No EventSystem found in the scene, and no UI Toolkit element is at ({inputPos.x:F1}, {inputPos.y:F1}). Ensure an EventSystem GameObject exists for uGUI.")); + } + + return (new UguiPressTarget(ResolvedPointerTargets.Empty, uguiPointerData), null); + } + + if (toolkit != null && (!uguiHasRaycastHit || !IsUguiHitInFront(uguiPointerData!, toolkit))) + { + return (new UIToolkitPressTarget(toolkit, inputPos, (int)parameters.Button), null); + } + + return (new UguiPressTarget(uguiResolved, uguiPointerData), null); + } + + private static (IMouseUiPressTarget? Target, SimulateMouseUiResponse? Failure) ResolveBypass( + MouseUiSimulationCommand parameters, + EventSystem? eventSystem, + Vector2 inputPos, + Vector2 screenPos, + MouseAction action) + { + if (eventSystem == null) + { + return (null, MouseUiSimulationResponseFactory.CreateFailure(parameters, NoEventSystemMessage)); + } + + PointerEventData pointerData = MouseUiPointerTargetResolver.CreatePointerPressData(eventSystem, screenPos, parameters.Button); + ResolvedPointerTargets resolved = MouseUiPointerTargetResolver.ResolvePressablePointerTargets( + parameters, eventSystem, inputPos, screenPos, pointerData, action); + SimulateMouseUiResponse? failure = resolved.FailureResponse; + if (failure == null) + { + return (new UguiPressTarget(resolved, pointerData), null); + } + + // Annotation paths of UI Toolkit controls contain '#', which no GameObject path lookup can match. + if (failure.Message.EndsWith("was not found.", System.StringComparison.Ordinal) && parameters.TargetPath.Contains("#")) + { + failure.Message += UIToolkitPathHint; + } + + return (null, failure); + } + + // The press data carries the uGUI raycast hit, whose raycaster decides the order against the panel. + private static bool IsUguiHitInFront(PointerEventData uguiPointerData, UIToolkitPointerTarget toolkit) + { + BaseRaycaster raycaster = uguiPointerData.pointerPressRaycast.module; + return MouseUiHitPriority.IsUguiInFront(raycaster.sortOrderPriority, toolkit.SortingOrder); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta new file mode 100644 index 000000000..5cc46dbef --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiPressTargetResolver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 94dd6bfdafc504163afebc155ba0247d +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs index 0f8e5c920..4eb4db168 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationResponseFactory.cs @@ -75,10 +75,12 @@ internal static SimulateMouseUiResponse CreateClickResult( Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}'" - : $"Clicked '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1})" + : $"Clicked {DescribeHit(hit)} at ({inputPos.x:F1}, {inputPos.y:F1})" : $"Clicked at ({inputPos.x:F1}, {inputPos.y:F1}) - no UI element hit", Action = MouseAction.Click.ToString(), HitGameObjectName = targetName, + UiSystem = hit.UiSystem, + HitPath = hit.Path, PositionX = inputPos.x, PositionY = inputPos.y }; @@ -96,15 +98,24 @@ internal static SimulateMouseUiResponse CreateLongPressResult( Message = hit.HasHit ? parameters.BypassRaycast ? $"Bypass-long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) via '{parameters.TargetPath}' for {parameters.Duration:F1}s" - : $"Long-pressed '{targetName}' at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s" + : $"Long-pressed {DescribeHit(hit)} at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s" : $"Long-pressed at ({inputPos.x:F1}, {inputPos.y:F1}) for {parameters.Duration:F1}s - no UI element hit", Action = MouseAction.LongPress.ToString(), HitGameObjectName = targetName, + UiSystem = hit.UiSystem, + HitPath = hit.Path, PositionX = inputPos.x, PositionY = inputPos.y }; } + private static string DescribeHit(MouseUiHitReport hit) + { + return hit.UiSystem == MouseUiHitSystem.UIToolkit + ? $"UI Toolkit element '{hit.Name}'" + : $"'{hit.Name}'"; + } + internal static SimulateMouseUiResponse CreateDragResult( MouseUiSimulationCommand parameters, Vector2 inputStart, diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs index 239600f46..4417ed508 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/MouseUiSimulationValidator.cs @@ -1,6 +1,4 @@ #nullable enable -using UnityEngine.EventSystems; - using io.github.hatayama.UnityCliLoop.Runtime; namespace io.github.hatayama.UnityCliLoop.FirstPartyTools @@ -12,7 +10,6 @@ internal static class MouseUiSimulationValidator { internal static SimulateMouseUiResponse? ValidateSimulationStart( MouseUiSimulationCommand parameters, - EventSystem? eventSystem, string pausedActionDescription) { PlayModeToolPreflightResult playModeResult = @@ -22,13 +19,6 @@ internal static class MouseUiSimulationValidator return MouseUiSimulationResponseFactory.CreatePreflightFailure(parameters, playModeResult); } - if (eventSystem == null) - { - return MouseUiSimulationResponseFactory.CreateFailure( - parameters, - "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."); - } - return ValidateSimulationRequestOptions(parameters); } @@ -91,7 +81,7 @@ private static bool RequiresBypassTargetPath(MouseAction action) || action == MouseAction.DragStart; } - private static bool IsDragAction(MouseAction action) + internal static bool IsDragAction(MouseAction action) { return action == MouseAction.Drag || action == MouseAction.DragStart diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs index 1c7354dc7..a4127095c 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiResponse.cs @@ -13,7 +13,24 @@ public class SimulateMouseUiResponse : UnityCliLoopToolResponse { public string Message { get; set; } = ""; public string Action { get; set; } = ""; + /// + /// Name of what received the press: the uGUI or physics GameObject that handles it, or the UI Toolkit + /// control under the point (its name, or its type name when it has none). Null when nothing was hit. + /// public string? HitGameObjectName { get; set; } + + /// + /// Which UI system was hit: "UGUI" for a target under a Canvas, "" for a target outside any Canvas (a + /// physics hit), "UIToolkit" for a UI Toolkit control, or null when nothing was hit. + /// + public string? UiSystem { get; set; } + + /// + /// Path of what was hit, in the form screenshot annotations use: the GameObject's hierarchy path for + /// uGUI and physics, or the document path, '#', and element chain for UI Toolkit. Null when nothing was + /// hit or when a UI Toolkit element belongs to no document, such as an open dropdown's menu. + /// + public string? HitPath { get; set; } public float PositionX { get; set; } public float PositionY { get; set; } public float? EndPositionX { get; set; } diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs index 1a711654f..948a55842 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/SimulateMouseUiUseCase.cs @@ -50,14 +50,20 @@ public async Task ExecuteAsync( EventSystem? eventSystem = EventSystem.current; SimulateMouseUiResponse? validationFailure = MouseUiSimulationValidator.ValidateSimulationStart( parameters, - eventSystem, PausedActionDescription); if (validationFailure != null) { return validationFailure; } - Debug.Assert(eventSystem != null, "ValidateSimulationStart must reject a missing EventSystem."); - EventSystem activeEventSystem = eventSystem!; + + // Clicks and long presses can reach UI Toolkit without an EventSystem and check for one when they + // resolve their target. DragMove and DragEnd executors never receive it, so drags are checked here. + if (MouseUiSimulationValidator.IsDragAction(parameters.Action) && eventSystem == null) + { + return MouseUiSimulationResponseFactory.CreateFailure( + parameters, + "No EventSystem found in the scene. Ensure an EventSystem GameObject exists."); + } MouseUiSimulationActivityLogger.LogSimulationStart(parameters, correlationId); OverlayCanvasFactory.EnsureExists(); @@ -70,7 +76,7 @@ public async Task ExecuteAsync( } SimulateMouseUiResponse response = - await ExecuteMouseAction(parameters, activeEventSystem, ct).ConfigureAwait(false); + await ExecuteMouseAction(parameters, eventSystem, ct).ConfigureAwait(false); MouseUiSimulationActivityLogger.LogSimulationComplete(parameters, response, correlationId); return response; @@ -78,7 +84,7 @@ public async Task ExecuteAsync( private async Task ExecuteMouseAction( MouseUiSimulationCommand parameters, - EventSystem eventSystem, + EventSystem? eventSystem, CancellationToken ct) { switch (parameters.Action) @@ -91,16 +97,18 @@ private async Task ExecuteMouseAction( ct).ConfigureAwait(false); case MouseAction.Drag: + Debug.Assert(eventSystem != null, "Drags without an EventSystem must be rejected before dispatch."); return await MouseUiOneShotDragExecutor.ExecuteDragOneShot( parameters, - eventSystem, + eventSystem!, _mainThreadCleanupScheduler, ct).ConfigureAwait(false); case MouseAction.DragStart: + Debug.Assert(eventSystem != null, "Drags without an EventSystem must be rejected before dispatch."); return await MouseUiIncrementalDragExecutor.ExecuteDragStart( parameters, - eventSystem, + eventSystem!, _mainThreadCleanupScheduler, ct).ConfigureAwait(false); diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs new file mode 100644 index 000000000..318bbd1a8 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs @@ -0,0 +1,42 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Delivers a click or long press to a UI Toolkit control by sending pointer events to its panel at the + /// requested Game View point. + /// + internal sealed class UIToolkitPressTarget : IMouseUiPressTarget + { + private readonly UIToolkitPointerTarget _target; + private readonly Vector2 _gameViewTopLeft; + private readonly int _button; + + internal UIToolkitPressTarget(UIToolkitPointerTarget target, Vector2 gameViewTopLeft, int button) + { + _target = target; + _gameViewTopLeft = gameViewTopLeft; + _button = button; + Report = new MouseUiHitReport(target.Name, target.Path, MouseUiHitSystem.UIToolkit); + } + + public MouseUiHitReport Report { get; } + + public void Click() + { + _target.PressDown(_gameViewTopLeft, _button); + _target.Release(_gameViewTopLeft, _button); + } + + public void PressDown() + { + _target.PressDown(_gameViewTopLeft, _button); + } + + public void Release() + { + _target.Release(_gameViewTopLeft, _button); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta new file mode 100644 index 000000000..727772e4b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UIToolkitPressTarget.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 2a98e11783c0f44ebaaffe60ea0219d2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs index 629acfa36..ada219b6d 100644 --- a/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs +++ b/Packages/src/Editor/FirstPartyTools/SimulateMouseUi/UguiPressTarget.cs @@ -1,4 +1,5 @@ #nullable enable +using UnityEngine; using UnityEngine.EventSystems; namespace io.github.hatayama.UnityCliLoop.FirstPartyTools @@ -22,11 +23,26 @@ internal UguiPressTarget(ResolvedPointerTargets resolvedTargets, PointerEventDat // UnityEngine.Object targets for null is not safe. bool shouldReleasePointer = resolvedTargets.RawTarget != null && resolvedTargets.Target != null; _shouldReleasePointer = shouldReleasePointer; - Report = new MouseUiHitReport(resolvedTargets.Target?.name, null, null); + Report = CreateReport(resolvedTargets.Target); } public MouseUiHitReport Report { get; } + // Decided by whether the handler sits under a Canvas rather than by raycaster type, because a bypass + // press has no raycaster and both paths must report alike. + private static MouseUiHitReport CreateReport(GameObject? target) + { + if (target == null) + { + return MouseUiHitReport.None; + } + + string uiSystem = target.GetComponentInParent() != null + ? MouseUiHitSystem.UGUI + : MouseUiHitSystem.NonUi; + return new MouseUiHitReport(target.name, GameObjectPathUtility.GetFullPath(target), uiSystem); + } + public void Click() { if (_resolvedTargets.RawTarget == null) From 05d10fcc4a5d5e40c5f0deb4304ce330ef51e928 Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:50:41 +0900 Subject: [PATCH 5/8] test: E2E presses UI Toolkit controls with simulate-mouse-ui at annotated points The UI Toolkit annotation E2E now drives simulate-mouse-ui at the coordinates the elements-only capture reports, in both the POSIX and PowerShell scripts: - a click on a button reports UIToolkit, its annotation path and name, and a click on a scrollbar reports the scrollbar, not a part inside it - a click on the covered part of a button reaches the panel in front, and a click at the annotated point reaches the button - a disabled button receives the press without reacting - a long press clicks once on release - the uGUI button still reacts and reports UGUI - a click still works after the EventSystem is disabled The existing simulate-mouse-input click stays as its own step. The listener and EventSystem changes are made in PlayMode only and go away when the script stops PlayMode. --- .../test-screenshot-uitoolkit-annotation.ps1 | 243 ++++++++++++++++-- .../test-screenshot-uitoolkit-annotation.sh | 199 ++++++++++++-- 2 files changed, 391 insertions(+), 51 deletions(-) diff --git a/scripts/test-screenshot-uitoolkit-annotation.ps1 b/scripts/test-screenshot-uitoolkit-annotation.ps1 index db01ae543..a5371d6d4 100644 --- a/scripts/test-screenshot-uitoolkit-annotation.ps1 +++ b/scripts/test-screenshot-uitoolkit-annotation.ps1 @@ -1,7 +1,8 @@ <# -E2E verification that screenshot --annotate-elements lists UI Toolkit controls, using the uloop CLI against -UIToolkitAnnotationDemoScene. Safe to run repeatedly: it stops PlayMode and restores the Game View size on exit, -and never saves the scene. +E2E verification that screenshot --annotate-elements lists UI Toolkit controls and that simulate-mouse-ui and +simulate-mouse-input press them at the annotated points, using the uloop CLI against UIToolkitAnnotationDemoScene. +Safe to run repeatedly: it stops PlayMode and restores the Game View size on exit, and never saves the scene, so +the listener and EventSystem changes made in PlayMode are discarded. Usage: powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\test-screenshot-uitoolkit-annotation.ps1 @@ -376,7 +377,8 @@ function Test-ElementsOnlyCapture { ) [object[]]$first = Get-UiToolkitElements -Response $AnnotatedResponse - [object[]]$second = Get-UiToolkitElements -Response (Invoke-ElementsOnlyCapture) + [pscustomobject]$elementsOnly = Invoke-ElementsOnlyCapture + [object[]]$second = Get-UiToolkitElements -Response $elementsOnly [string]$firstPaths = (@($first | ForEach-Object { $_.Path }) | Sort-Object) -join "|" [string]$secondPaths = (@($second | ForEach-Object { $_.Path }) | Sort-Object) -join "|" Assert-EqualText -Actual $secondPaths -Expected $firstPaths -Context "UI Toolkit element paths in the elements-only capture" @@ -387,48 +389,219 @@ function Test-ElementsOnlyCapture { Assert-Near -Actual $element.$field -Expected $match.$field -Context "$($element.Path) $field" } } + + return $elementsOnly } -function Get-PlayClickCount { +function Get-ClickCount { + param( + [string]$Name + ) + [string]$code = @" UIToolkitAnnotationDemo demo = UnityEngine.Object.FindObjectOfType(); if (demo == null) { return "ERROR: UIToolkitAnnotationDemo not found"; } -return demo.GetClickCount("play"); +return demo.GetClickCount("$Name"); "@ [pscustomobject]$result = Invoke-UloopJson -CommandArguments @("execute-dynamic-code", "--code", $code) - Assert-Condition -Condition ($result.Success -eq $true) -Message "Read play click count failed: $($result.ErrorMessage)" + Assert-Condition -Condition ($result.Success -eq $true) -Message "Read $Name click count failed: $($result.ErrorMessage)" [string]$count = [string]$result.Result if ($count.StartsWith("ERROR:")) { - throw "Read play click count failed: $($count.Substring(6).Trim())" + throw "Read $Name click count failed: $($count.Substring(6).Trim())" } return $count } -# simulate-mouse-input reaches UI Toolkit buttons through the EventSystem; simulate-mouse-ui does not. -function Test-Click { +function Invoke-DynamicCodeText { param( - [pscustomobject]$AnnotatedResponse + [string]$Code, + [string]$Context ) - [object[]]$elements = Get-UiToolkitElements -Response $AnnotatedResponse - [pscustomobject]$play = @($elements | Where-Object { $_.Name -eq "play" })[0] - [int]$before = [int](Get-PlayClickCount) + [pscustomobject]$result = Invoke-UloopJson -CommandArguments @("execute-dynamic-code", "--code", $Code) + Assert-Condition -Condition ($result.Success -eq $true) -Message "$Context failed: $($result.ErrorMessage)" + return [string]$result.Result +} + +function Format-Coordinate { + param( + [double]$Value + ) + + return $Value.ToString([Globalization.CultureInfo]::InvariantCulture) +} + +function Get-AnnotatedElement { + param( + [pscustomobject]$Response, + [string]$UiSystem, + [string]$Name + ) + + return @($Response.Screenshots[0].AnnotatedElements | Where-Object { $_.UiSystem -eq $UiSystem -and $_.Name -eq $Name })[0] +} + +# Returns the simulate-mouse-ui response after checking it succeeded. +function Invoke-SimulateUi { + param( + [string]$Context, + [string[]]$Arguments + ) + + [pscustomobject]$response = Invoke-UloopJson -CommandArguments (@("simulate-mouse-ui") + $Arguments) + Assert-Condition -Condition ($response.Success -eq $true) -Message "$Context failed: $($response.Message)" + return $response +} + +function Invoke-SimulateUiClick { + param( + [string]$Context, + [double]$X, + [double]$Y + ) + + return Invoke-SimulateUi -Context $Context -Arguments @("--action", "Click", "--x", (Format-Coordinate $X), "--y", (Format-Coordinate $Y)) +} + +# simulate-mouse-ui reaches the UI Toolkit button directly and reports it, including for a click inside a +# scrollbar, where it must report the scrollbar rather than the part inside it. +function Test-UiClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") + + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on play" -X $play.SimX -Y $play.SimY + Assert-EqualText -Actual $response.UiSystem -Expected "UIToolkit" -Context "simulate-mouse-ui click on play UiSystem" + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/BackDocument#play" -Context "simulate-mouse-ui click on play HitPath" + Assert-EqualText -Actual $response.HitGameObjectName -Expected "play" -Context "simulate-mouse-ui click on play HitGameObjectName" + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after simulate-mouse-ui" + + [pscustomobject]$scrollbar = @($ElementsOnly.Screenshots[0].AnnotatedElements | Where-Object { $_.UiSystem -eq "UIToolkit" -and $_.Type -eq "Scrollbar" })[0] + $response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on the scrollbar" -X $scrollbar.SimX -Y $scrollbar.SimY + Assert-EqualText -Actual $response.HitPath -Expected $scrollbar.Path -Context "simulate-mouse-ui click on the scrollbar HitPath" +} + +# simulate-mouse-input reaches UI Toolkit buttons through the EventSystem's Input System UI module. +function Test-InputClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") [pscustomobject]$response = Invoke-UloopJson -CommandArguments @( "simulate-mouse-input", "--action", "Click", "--x", - ([double]$play.SimX).ToString([Globalization.CultureInfo]::InvariantCulture), + (Format-Coordinate $play.SimX), "--y", - ([double]$play.SimY).ToString([Globalization.CultureInfo]::InvariantCulture) + (Format-Coordinate $play.SimY) ) - Assert-Condition -Condition ($response.Success -eq $true) -Message "Click play failed: $($response.Message)" + Assert-Condition -Condition ($response.Success -eq $true) -Message "simulate-mouse-input click on play failed: $($response.Message)" Start-Sleep -Seconds 1 - Assert-EqualText -Actual (Get-PlayClickCount) -Expected ([string]($before + 1)) -Context "play click count" + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after simulate-mouse-input" +} + +# The front panel takes a click on the covered part; the uncovered annotated point reaches the button. +function Test-CoveredClick { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$partlyCovered = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "partly-covered" + [double]$coveredX = $partlyCovered.BoundsMinX + ($partlyCovered.BoundsMaxX - $partlyCovered.BoundsMinX) * 0.3 + [double]$coveredY = ($partlyCovered.BoundsMinY + $partlyCovered.BoundsMaxY) / 2 + [string]$before = Get-ClickCount -Name "partly-covered" + + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on the covered part" -X $coveredX -Y $coveredY + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/FrontDocument#cover" -Context "simulate-mouse-ui click on the covered part HitPath" + Assert-EqualText -Actual (Get-ClickCount -Name "partly-covered") -Expected $before -Context "partly-covered click count after clicking its cover" + + Invoke-SimulateUiClick -Context "simulate-mouse-ui click on partly-covered" -X $partlyCovered.SimX -Y $partlyCovered.SimY | Out-Null + Assert-EqualText -Actual (Get-ClickCount -Name "partly-covered") -Expected ([string]([int]$before + 1)) -Context "partly-covered click count" +} + +# The disabled button is not annotated, so its layout centre in the demo (100,440 200x50) is used. The press is +# delivered but, as with real input, the disabled button does not react. +function Test-DisabledClick { + [pscustomobject]$response = Invoke-SimulateUiClick -Context "simulate-mouse-ui click on locked" -X 200 -Y 465 + Assert-EqualText -Actual $response.HitPath -Expected "UIToolkitAnnotationDemo/BackDocument#locked" -Context "simulate-mouse-ui click on locked HitPath" + Assert-EqualText -Actual (Get-ClickCount -Name "locked") -Expected "0" -Context "locked click count" +} + +function Test-LongPress { + param( + [pscustomobject]$ElementsOnly + ) + + [pscustomobject]$play = Get-AnnotatedElement -Response $ElementsOnly -UiSystem "UIToolkit" -Name "play" + [int]$before = [int](Get-ClickCount -Name "play") + + Invoke-SimulateUi -Context "simulate-mouse-ui long press on play" -Arguments @( + "--action", "LongPress", "--duration", "0.5", "--x", (Format-Coordinate $play.SimX), "--y", (Format-Coordinate $play.SimY)) | Out-Null + + Assert-EqualText -Actual (Get-ClickCount -Name "play") -Expected ([string]($before + 1)) -Context "play click count after long press" +} + +# The demo does not count uGUI clicks, so a listener added in PlayMode writes the count into the button label. +function Test-UguiClick { + param( + [pscustomobject]$ElementsOnly + ) + + [string]$listenerCode = @' +using UnityEngine; +using UnityEngine.UI; +GameObject buttonObject = GameObject.Find("ugui-button"); +if (buttonObject == null) { return "ERROR: ugui-button not found"; } +Text label = buttonObject.GetComponentInChildren(); +if (label == null) { return "ERROR: ugui-button has no Text"; } +int clicks = 0; +buttonObject.GetComponent