From f7a88b64176c87b2b4f5db41800ba408b3c80cdf Mon Sep 17 00:00:00 2001 From: hatayama Date: Wed, 14 Oct 2026 00:11:54 +0900 Subject: [PATCH] refactor: move UI Toolkit screen elements into a shared module simulate-mouse-ui is about to find and press UI Toolkit controls the same way screenshot annotations list them, and one tool may not reference another. The panel gathering, picking, element paths and control walk therefore move out of the Screenshot assembly into Common.UIToolkit, behind a single public entry point that works in top-left Game View pixels. Its parts stay internal and are visible only to the tests. Screenshot keeps UIToolkitElementCollector.Collect with the same signature as an adapter: it supplies the uGUI probe points and turns the results back into bottom-left annotation entries with the same conversions in the same order, so annotations are unchanged. --- .../Editor/UnityCLILoop.Tests.Editor.asmdef | 1 + .../UnityCLILoop.Tests.PlayMode.asmdef | 1 + .../Common/MouseUi/AssemblyInfo.cs | 1 + .../FirstPartyTools/Common/UIToolkit.meta | 8 + .../Common/UIToolkit/AssemblyInfo.cs | 6 + .../Common/UIToolkit/AssemblyInfo.cs.meta | 11 + .../UIToolkit/UIToolkitControlCollector.cs | 141 ++++++++ .../UIToolkitControlCollector.cs.meta | 11 + .../UIToolkit}/UIToolkitElementClassifier.cs | 0 .../UIToolkitElementClassifier.cs.meta | 0 .../Common/UIToolkit/UIToolkitElementPath.cs | 43 +++ .../UIToolkit/UIToolkitElementPath.cs.meta | 11 + .../UIToolkit}/UIToolkitPanelKind.cs | 0 .../UIToolkit}/UIToolkitPanelKind.cs.meta | 0 .../UIToolkit}/UIToolkitReachabilityProbe.cs | 0 .../UIToolkitReachabilityProbe.cs.meta | 0 .../UIToolkit/UIToolkitScreenControl.cs | 49 +++ .../UIToolkit/UIToolkitScreenControl.cs.meta | 11 + .../UIToolkit/UIToolkitScreenElements.cs | 36 ++ .../UIToolkit/UIToolkitScreenElements.cs.meta | 11 + .../UIToolkit}/UIToolkitScreenMapping.cs | 0 .../UIToolkit}/UIToolkitScreenMapping.cs.meta | 0 .../Common/UIToolkit/UIToolkitScreenPanels.cs | 137 ++++++++ .../UIToolkit/UIToolkitScreenPanels.cs.meta | 11 + ...tPartyTools.Common.UIToolkit.Editor.asmdef | 18 + ...yTools.Common.UIToolkit.Editor.asmdef.meta | 7 + .../UIToolkitElementCollector.cs | 320 ++---------------- ...p.FirstPartyTools.Screenshot.Editor.asmdef | 1 + 28 files changed, 546 insertions(+), 289 deletions(-) create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs.meta rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitElementClassifier.cs (100%) rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitElementClassifier.cs.meta (100%) create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs.meta rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitPanelKind.cs (100%) rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitPanelKind.cs.meta (100%) rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitReachabilityProbe.cs (100%) rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitReachabilityProbe.cs.meta (100%) create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs.meta rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitScreenMapping.cs (100%) rename Packages/src/Editor/FirstPartyTools/{Screenshot/UIToolkitAnnotation => Common/UIToolkit}/UIToolkitScreenMapping.cs.meta (100%) create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs.meta create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef create mode 100644 Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef.meta diff --git a/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef b/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef index 6963c78b6d..457d9307bb 100644 --- a/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef +++ b/Assets/Tests/Editor/UnityCLILoop.Tests.Editor.asmdef @@ -17,6 +17,7 @@ "GUID:f1a38fcd7e85047c5a86331a3e37ddfc", "GUID:45d3617c96fa64d3e95383450e67f25a", "GUID:a2d87883023de4cf59b7d1962d5dd8aa", + "GUID:569bba870e5e64d0d94144057b927001", "GUID:527f26a36b5043c2bd4d4036d04cd76d", "GUID:94d8abc693f543a691a4645a5ff42e5c", "GUID:aa7cf56cc5f074e57ba35272b877e116", diff --git a/Assets/Tests/PlayMode/UnityCLILoop.Tests.PlayMode.asmdef b/Assets/Tests/PlayMode/UnityCLILoop.Tests.PlayMode.asmdef index f3c297c562..2ed54bbd2e 100644 --- a/Assets/Tests/PlayMode/UnityCLILoop.Tests.PlayMode.asmdef +++ b/Assets/Tests/PlayMode/UnityCLILoop.Tests.PlayMode.asmdef @@ -14,6 +14,7 @@ "GUID:45d3617c96fa64d3e95383450e67f25a", "GUID:527f26a36b5043c2bd4d4036d04cd76d", "GUID:a2d87883023de4cf59b7d1962d5dd8aa", + "GUID:569bba870e5e64d0d94144057b927001", "GUID:aa7cf56cc5f074e57ba35272b877e116", "GUID:dc412b2cd951d452b8dcec99c3adeb19", "GUID:3b7aa330d87dc452293296dd5830859b", diff --git a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/AssemblyInfo.cs b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/AssemblyInfo.cs index 3cce180a5f..2d7abfd1a7 100644 --- a/Packages/src/Editor/FirstPartyTools/Common/MouseUi/AssemblyInfo.cs +++ b/Packages/src/Editor/FirstPartyTools/Common/MouseUi/AssemblyInfo.cs @@ -2,6 +2,7 @@ [assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.Editor")] [assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.Common.InputRecording.Editor")] +[assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor")] [assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.ReplayInput.Editor")] [assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.Screenshot.Editor")] [assembly: InternalsVisibleTo("UnityCLILoop.FirstPartyTools.SimulateMouseUi.Editor")] diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit.meta new file mode 100644 index 0000000000..93e15e8c20 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 0762c3778edd0468ca0bd7dd96a36017 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs new file mode 100644 index 0000000000..6f9143d26e --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs @@ -0,0 +1,6 @@ +using System.Runtime.CompilerServices; + +// Only the tests see these internals: tools use this module through its entry point alone, and the +// compiler keeps them away from the parts behind it. +[assembly: InternalsVisibleTo("UnityCLILoop.Tests.Editor")] +[assembly: InternalsVisibleTo("UnityCLILoop.Tests.PlayMode")] diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs.meta new file mode 100644 index 0000000000..fdfcb15522 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/AssemblyInfo.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: e4f778f9f51ce401bb91136ac63086a4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs new file mode 100644 index 0000000000..61d742d76b --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs @@ -0,0 +1,141 @@ +#nullable enable +using System; +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Walks the documents of the gathered panels and lists their reachable controls in top-left Game View + /// pixels, skipping controls that pointer input could not reach. + /// + internal static class UIToolkitControlCollector + { + // Expects the panels' layouts to be settled already, because bounds and picking read them. + internal static List Collect( + List panels, + Func> probePointsForTopLeftBounds) + { + List controls = new List(); + foreach (UIToolkitScreenPanels.PanelEntry entry in panels) + { + if (entry.Mapping == null) + { + continue; + } + + VisitContext context = new VisitContext( + entry, + entry.Mapping, + UIToolkitScreenPanels.CreateFrontPickers(panels, entry), + point => UIToolkitScreenPanels.PickInPanel(entry.Panel, point), + probePointsForTopLeftBounds, + controls); + foreach (UIDocument document in entry.Documents) + { + Visit(document.rootVisualElement, document, context); + } + } + + return controls; + } + + private static void Visit(VisualElement element, UIDocument document, VisitContext context) + { + // A child can still resolve to Flex under a display:none parent, so the whole subtree stops here. + if (element.resolvedStyle.display == DisplayStyle.None) + { + return; + } + + string? type = UIToolkitElementClassifier.Classify(element); + if (type != null) + { + TryAdd(element, type, document, context); + + // The parts inside a control (the slider and buttons of a Scroller, for example) are not + // separate controls. + return; + } + + // hierarchy, not Children(): Children() lists only the content container, which hides a + // ScrollView's scrollers. + for (int index = 0; index < element.hierarchy.childCount; index++) + { + Visit(element.hierarchy[index], document, context); + } + } + + private static void TryAdd(VisualElement element, string type, UIDocument document, VisitContext context) + { + if (!element.enabledInHierarchy || element.resolvedStyle.visibility == Visibility.Hidden) + { + return; + } + + Rect worldBound = element.worldBound; + if (!IsFiniteNonEmpty(worldBound)) + { + return; + } + + Vector2 topLeftMin = context.Mapping.PanelToScreenTopLeft(new Vector2(worldBound.xMin, worldBound.yMin)); + Vector2 topLeftMax = context.Mapping.PanelToScreenTopLeft(new Vector2(worldBound.xMax, worldBound.yMax)); + Rect topLeftBounds = Rect.MinMaxRect(topLeftMin.x, topLeftMin.y, topLeftMax.x, topLeftMax.y); + + Vector2? reachable = UIToolkitReachabilityProbe.FindReachablePoint( + context.ProbePointsForTopLeftBounds(topLeftBounds), element, context.FrontPickers, context.OwnPicker); + if (reachable == null) + { + return; + } + + VisualElement? parent = element.hierarchy.parent; + context.Controls.Add(new UIToolkitScreenControl( + UIToolkitElementPath.GetDisplayName(element), + UIToolkitElementPath.Build(document, element), + type, + topLeftBounds, + reachable.Value, + Mathf.FloorToInt(context.Entry.SortingOrder), + parent == null ? 0 : parent.hierarchy.IndexOf(element))); + } + + private static bool IsFiniteNonEmpty(Rect rect) + { + return float.IsFinite(rect.x) + && float.IsFinite(rect.y) + && float.IsFinite(rect.width) + && float.IsFinite(rect.height) + && rect.width > 0f + && rect.height > 0f; + } + + private sealed class VisitContext + { + internal VisitContext( + UIToolkitScreenPanels.PanelEntry entry, + UIToolkitScreenMapping mapping, + List> frontPickers, + Func ownPicker, + Func> probePointsForTopLeftBounds, + List controls) + { + Entry = entry; + Mapping = mapping; + FrontPickers = frontPickers; + OwnPicker = ownPicker; + ProbePointsForTopLeftBounds = probePointsForTopLeftBounds; + Controls = controls; + } + + internal UIToolkitScreenPanels.PanelEntry Entry { get; } + internal UIToolkitScreenMapping Mapping { get; } + internal List> FrontPickers { get; } + internal Func OwnPicker { get; } + internal Func> ProbePointsForTopLeftBounds { get; } + internal List Controls { get; } + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs.meta new file mode 100644 index 0000000000..e6e5e54847 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitControlCollector.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a6afaba0bcc1a45978ec99510bea2198 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementClassifier.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementClassifier.cs similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementClassifier.cs rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementClassifier.cs diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementClassifier.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementClassifier.cs.meta similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementClassifier.cs.meta rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementClassifier.cs.meta diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs new file mode 100644 index 0000000000..2fe140ba06 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs @@ -0,0 +1,43 @@ +#nullable enable +using System.Collections.Generic; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Names UI Toolkit elements the way annotations report them: the owning document's GameObject path, + /// a '#', then the element chain below the document root. + /// + internal static class UIToolkitElementPath + { + private const char PATH_SEPARATOR = '#'; + private const string CHAIN_SEPARATOR = "/"; + + internal static string Build(UIDocument document, VisualElement element) + { + return GameObjectPathUtility.GetFullPath(document.gameObject) + PATH_SEPARATOR + + BuildElementChain(document.rootVisualElement, element); + } + + internal static string GetDisplayName(VisualElement element) + { + return string.IsNullOrEmpty(element.name) ? element.GetType().Name : element.name; + } + + // Lists the elements from just below the document root down to the element, so the root itself, which + // has no name of its own, is left out. + private static string BuildElementChain(VisualElement root, VisualElement element) + { + List segments = new List(); + VisualElement? current = element; + while (current != null && current != root) + { + segments.Add(GetDisplayName(current)); + current = current.hierarchy.parent; + } + + segments.Reverse(); + return string.Join(CHAIN_SEPARATOR, segments); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs.meta new file mode 100644 index 0000000000..9153987e20 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitElementPath.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 0063bf7119a134d3482034492d38a1b7 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitPanelKind.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPanelKind.cs similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitPanelKind.cs rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPanelKind.cs diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitPanelKind.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPanelKind.cs.meta similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitPanelKind.cs.meta rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitPanelKind.cs.meta diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitReachabilityProbe.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitReachabilityProbe.cs similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitReachabilityProbe.cs rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitReachabilityProbe.cs diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitReachabilityProbe.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitReachabilityProbe.cs.meta similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitReachabilityProbe.cs.meta rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitReachabilityProbe.cs.meta diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs new file mode 100644 index 0000000000..c8d69dfe20 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs @@ -0,0 +1,49 @@ +#nullable enable +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// One interactive UI Toolkit control on screen, in top-left Game View pixels. + /// + public sealed class UIToolkitScreenControl + { + public UIToolkitScreenControl( + string name, + string path, + string type, + Rect topLeftBounds, + Vector2 topLeftReachablePoint, + int sortingOrder, + int siblingIndex) + { + Name = name; + Path = path; + Type = type; + TopLeftBounds = topLeftBounds; + TopLeftReachablePoint = topLeftReachablePoint; + SortingOrder = sortingOrder; + SiblingIndex = siblingIndex; + } + + /// The element's name, or its type name when it has none. + public string Name { get; } + + /// The owning document's GameObject path, '#', and the element chain below the document root. + public string Path { get; } + + /// The kind of control, such as "Button", "Toggle" or "Slider". + public string Type { get; } + + public Rect TopLeftBounds { get; } + + /// A point inside the bounds where pointer input reaches this control. + public Vector2 TopLeftReachablePoint { get; } + + /// The panel's sorting order, rounded down. + public int SortingOrder { get; } + + /// The control's index among its parent's children. + public int SiblingIndex { get; } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs.meta new file mode 100644 index 0000000000..6f7cb1efea --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenControl.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 986bdff0569264bf395eed0b8b042fe8 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs new file mode 100644 index 0000000000..f3351c5714 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs @@ -0,0 +1,36 @@ +#nullable enable +using System; +using System.Collections.Generic; +using UnityEngine; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Entry point for the UI Toolkit controls shown over the Game View on screen-space panels. Every point and + /// rectangle is in Game View pixels with the origin at the top left, the space screenshot annotations and + /// simulated pointer input use. + /// + public static class UIToolkitScreenElements + { + /// + /// Returns the enabled, visible controls that pointer input can reach, each with a point where it does. + /// Controls inside another control (the buttons of a scrollbar, for example) are not listed separately. + /// + /// + /// Gives the candidate points to try for a control's bounds, in the order to try them. The first point + /// that is not covered by a panel in front or by another element becomes the reachable point. + /// + public static IReadOnlyList CollectControls( + Func> probePointsForTopLeftBounds) + { + if (probePointsForTopLeftBounds == null) + { + throw new ArgumentNullException(nameof(probePointsForTopLeftBounds)); + } + + List panels = UIToolkitScreenPanels.Gather(); + UIToolkitScreenPanels.SettleLayouts(panels); + return UIToolkitControlCollector.Collect(panels, probePointsForTopLeftBounds); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs.meta new file mode 100644 index 0000000000..80e950e025 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenElements.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: d2e4ddae68bc647549c46ae552ba5cdf +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitScreenMapping.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenMapping.cs similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitScreenMapping.cs rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenMapping.cs diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitScreenMapping.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenMapping.cs.meta similarity index 100% rename from Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitScreenMapping.cs.meta rename to Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenMapping.cs.meta diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs new file mode 100644 index 0000000000..254ca21787 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs @@ -0,0 +1,137 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using UnityEngine.UIElements; + +namespace io.github.hatayama.UnityCliLoop.FirstPartyTools +{ + /// + /// Finds the screen-space UI Toolkit panels that show in the Game View or take input there, and picks + /// elements on them in top-left Game View pixels. + /// + internal static class UIToolkitScreenPanels + { + internal static List Gather() + { +#if UNITY_6000_4_OR_NEWER + UIDocument[] documents = UnityEngine.Object.FindObjectsByType(); +#else + UIDocument[] documents = UnityEngine.Object.FindObjectsOfType(); +#endif + List panels = new List(); + foreach (UIDocument document in documents) + { + IPanel? panel = GetScreenSpacePanelShownInGameView(document); + if (panel == null) + { + continue; + } + + PanelEntry? entry = panels.Find(candidate => candidate.Panel == panel); + if (entry == null) + { + entry = CreatePanelEntry(panel, document.panelSettings); + panels.Add(entry); + } + + entry.Documents.Add(document); + } + + return panels; + } + + // worldBound and visualTree.layout are only settled once the panel validates its layout, and Pick is + // the only public call that does so. Every panel is settled, including those used only to decide + // what covers what, because PickInPanel reads their layout too. + internal static void SettleLayouts(List panels) + { + foreach (PanelEntry entry in panels) + { + entry.Panel.Pick(Vector2.zero); + } + } + + // Panels with the same sorting order are not treated as covering each other: Unity breaks that tie by + // creation order, which is not public. + internal static List> CreateFrontPickers(List panels, PanelEntry entry) + { + return panels + .Where(other => other.SortingOrder > entry.SortingOrder) + .OrderByDescending(other => other.SortingOrder) + .Select(other => (Func)(point => PickInPanel(other.Panel, point))) + .ToList(); + } + + // Uses the forward screen-to-panel function Unity applies to input, so the result matches where input + // goes even for panels whose function cannot be inverted. + internal static VisualElement? PickInPanel(IPanel panel, Vector2 screenTopLeftPoint) + { + Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(panel, screenTopLeftPoint); + if (!panel.visualTree.layout.Contains(panelPoint)) + { + return null; + } + + return panel.Pick(panelPoint); + } + + // Returns null for documents that contribute nothing: their panel neither shows in the Game View nor + // takes input there, or they are nested and reached through their parent document's root. + private static IPanel? GetScreenSpacePanelShownInGameView(UIDocument document) + { + if (!document.isActiveAndEnabled || document.parentUI != null) + { + return null; + } + + PanelSettings settings = document.panelSettings; + VisualElement root = document.rootVisualElement; + if (settings == null || root == null || root.panel == null) + { + return null; + } + + if (settings.targetDisplay != Display.activeEditorGameViewTarget) + { + return null; + } + + // World-space panels do not cover overlay controls and their elements have no screen rectangle. + if (UIToolkitPanelKind.IsWorldSpace(settings)) + { + return null; + } + + return root.panel; + } + + // A panel that renders into a texture still takes screen input, so it stays as a cover candidate, but + // its controls are not on the screenshot and get no mapping. + private static PanelEntry CreatePanelEntry(IPanel panel, PanelSettings settings) + { + UIToolkitScreenMapping? mapping = settings.targetTexture == null + ? UIToolkitScreenMapping.TryCreate(screen => RuntimePanelUtils.ScreenToPanel(panel, screen)) + : null; + return new PanelEntry(panel, settings.sortingOrder, mapping); + } + + internal sealed class PanelEntry + { + internal PanelEntry(IPanel panel, float sortingOrder, UIToolkitScreenMapping? mapping) + { + Panel = panel; + SortingOrder = sortingOrder; + Mapping = mapping; + } + + internal IPanel Panel { get; } + internal float SortingOrder { get; } + + // Null when the panel's controls cannot be placed on the screenshot; the panel still covers others. + internal UIToolkitScreenMapping? Mapping { get; } + internal List Documents { get; } = new List(); + } + } +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs.meta new file mode 100644 index 0000000000..e06d7d0827 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UIToolkitScreenPanels.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: c1b1722f144d94fef89b4efcd6e14316 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef new file mode 100644 index 0000000000..525f80e291 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef @@ -0,0 +1,18 @@ +{ + "name": "UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor", + "rootNamespace": "io.github.hatayama.UnityCliLoop.FirstPartyTools", + "references": [ + "GUID:a2d87883023de4cf59b7d1962d5dd8aa" + ], + "includePlatforms": [ + "Editor" + ], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": false, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} diff --git a/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef.meta b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef.meta new file mode 100644 index 0000000000..2e5312c8e0 --- /dev/null +++ b/Packages/src/Editor/FirstPartyTools/Common/UIToolkit/UnityCLILoop.FirstPartyTools.Common.UIToolkit.Editor.asmdef.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 569bba870e5e64d0d94144057b927001 +AssemblyDefinitionImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementCollector.cs b/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementCollector.cs index 86fe5059b9..7df4588027 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementCollector.cs +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/UIToolkitAnnotation/UIToolkitElementCollector.cs @@ -1,9 +1,7 @@ #nullable enable -using System; using System.Collections.Generic; using System.Linq; using UnityEngine; -using UnityEngine.UIElements; using io.github.hatayama.UnityCliLoop.ToolContracts; @@ -16,309 +14,53 @@ namespace io.github.hatayama.UnityCliLoop.FirstPartyTools /// internal static class UIToolkitElementCollector { - private const char PATH_SEPARATOR = '#'; - private const string CHAIN_SEPARATOR = "/"; - + // gameViewHeight, not Screen.height: uloop commands run outside the player loop, where Screen reports the + // size of the last Editor view drawn. ConvertToSimCoordinates flips the result back with the Game View + // height, so the same height is used here. internal static void Collect(List elements, float gameViewHeight) { - List panels = GatherPanels(); - - // worldBound and visualTree.layout are only settled once the panel validates its layout, and Pick is - // the only public call that does so. Every panel is settled, including those used only to decide - // what covers what, because PickInPanel reads their layout too. - foreach (PanelEntry entry in panels) - { - entry.Panel.Pick(Vector2.zero); - } - - foreach (PanelEntry entry in panels) - { - if (entry.Mapping == null) - { - continue; - } - - VisitContext context = new VisitContext( - gameViewHeight, - entry, - entry.Mapping, - CreateFrontPickers(panels, entry), - point => PickInPanel(entry.Panel, point), - elements); - foreach (UIDocument document in entry.Documents) - { - Visit(document.rootVisualElement, document, context); - } - } - } - - private static List GatherPanels() - { -#if UNITY_6000_4_OR_NEWER - UIDocument[] documents = UnityEngine.Object.FindObjectsByType(); -#else - UIDocument[] documents = UnityEngine.Object.FindObjectsOfType(); -#endif - List panels = new List(); - foreach (UIDocument document in documents) + IReadOnlyList controls = UIToolkitScreenElements.CollectControls( + topLeftBounds => GenerateTopLeftProbePoints(topLeftBounds, gameViewHeight)); + foreach (UIToolkitScreenControl control in controls) { - IPanel? panel = GetScreenSpacePanelShownInGameView(document); - if (panel == null) - { - continue; - } - - PanelEntry? entry = panels.Find(candidate => candidate.Panel == panel); - if (entry == null) + Rect bottomLeftBounds = ToBottomLeft(control.TopLeftBounds, gameViewHeight); + elements.Add(new UIElementInfo { - entry = CreatePanelEntry(panel, document.panelSettings); - panels.Add(entry); - } - - entry.Documents.Add(document); - } - - return panels; - } - - // Returns null for documents that contribute nothing: their panel neither shows in the Game View nor - // takes input there, or they are nested and reached through their parent document's root. - private static IPanel? GetScreenSpacePanelShownInGameView(UIDocument document) - { - if (!document.isActiveAndEnabled || document.parentUI != null) - { - return null; - } - - PanelSettings settings = document.panelSettings; - VisualElement root = document.rootVisualElement; - if (settings == null || root == null || root.panel == null) - { - return null; - } - - if (settings.targetDisplay != Display.activeEditorGameViewTarget) - { - return null; - } - - // World-space panels do not cover overlay controls and their elements have no screen rectangle. - if (UIToolkitPanelKind.IsWorldSpace(settings)) - { - return null; - } - - return root.panel; - } - - // A panel that renders into a texture still takes screen input, so it stays as a cover candidate, but - // its controls are not on the screenshot and get no mapping. - private static PanelEntry CreatePanelEntry(IPanel panel, PanelSettings settings) - { - UIToolkitScreenMapping? mapping = settings.targetTexture == null - ? UIToolkitScreenMapping.TryCreate(screen => RuntimePanelUtils.ScreenToPanel(panel, screen)) - : null; - return new PanelEntry(panel, settings.sortingOrder, mapping); - } - - // Panels with the same sorting order are not treated as covering each other: Unity breaks that tie by - // creation order, which is not public. - private static List> CreateFrontPickers(List panels, PanelEntry entry) - { - return panels - .Where(other => other.SortingOrder > entry.SortingOrder) - .OrderByDescending(other => other.SortingOrder) - .Select(other => (Func)(point => PickInPanel(other.Panel, point))) - .ToList(); - } - - // Uses the forward screen-to-panel function Unity applies to input, so the result matches where input - // goes even for panels whose function cannot be inverted. - private static VisualElement? PickInPanel(IPanel panel, Vector2 screenTopLeftPoint) - { - Vector2 panelPoint = RuntimePanelUtils.ScreenToPanel(panel, screenTopLeftPoint); - if (!panel.visualTree.layout.Contains(panelPoint)) - { - return null; + Name = control.Name, + Path = control.Path, + Type = control.Type, + Interaction = UIElementAnnotationStyling.GetInteractionForType(control.Type), + UiSystem = AnnotatedUiSystem.UIToolkit, + SimX = control.TopLeftReachablePoint.x, + SimY = gameViewHeight - control.TopLeftReachablePoint.y, + BoundsMinX = bottomLeftBounds.xMin, + BoundsMinY = bottomLeftBounds.yMin, + BoundsMaxX = bottomLeftBounds.xMax, + BoundsMaxY = bottomLeftBounds.yMax, + SortingOrder = control.SortingOrder, + SiblingIndex = control.SiblingIndex + }); } - - return panel.Pick(panelPoint); - } - - private static void Visit(VisualElement element, UIDocument document, VisitContext context) - { - // A child can still resolve to Flex under a display:none parent, so the whole subtree stops here. - if (element.resolvedStyle.display == DisplayStyle.None) - { - return; - } - - string? type = UIToolkitElementClassifier.Classify(element); - if (type != null) - { - TryAdd(element, type, document, context); - - // The parts inside a control (the slider and buttons of a Scroller, for example) are not - // separate controls. - return; - } - - // hierarchy, not Children(): Children() lists only the content container, which hides a - // ScrollView's scrollers. - for (int index = 0; index < element.hierarchy.childCount; index++) - { - Visit(element.hierarchy[index], document, context); - } - } - - private static void TryAdd(VisualElement element, string type, UIDocument document, VisitContext context) - { - if (!element.enabledInHierarchy || element.resolvedStyle.visibility == Visibility.Hidden) - { - return; - } - - Rect worldBound = element.worldBound; - if (!IsFiniteNonEmpty(worldBound)) - { - return; - } - - // Not Screen.height: uloop commands run outside the player loop, where Screen reports the size of the - // last Editor view drawn. ConvertToSimCoordinates flips the result back with the Game View height, so - // the same height is used here. - float gameViewHeight = context.GameViewHeight; - Vector2 topLeftMin = context.Mapping.PanelToScreenTopLeft(new Vector2(worldBound.xMin, worldBound.yMin)); - Vector2 topLeftMax = context.Mapping.PanelToScreenTopLeft(new Vector2(worldBound.xMax, worldBound.yMax)); - Rect bottomLeftBounds = Rect.MinMaxRect( - topLeftMin.x, - gameViewHeight - topLeftMax.y, - topLeftMax.x, - gameViewHeight - topLeftMin.y); - - Vector2? reachable = FindReachablePoint(element, bottomLeftBounds, gameViewHeight, context); - if (reachable == null) - { - return; - } - - context.Elements.Add(CreateElementInfo( - element, type, document, context.Entry, bottomLeftBounds, new Vector2(reachable.Value.x, gameViewHeight - reachable.Value.y))); } // Probes the same points as the uGUI annotator, converted to the top-left origin panels take input in. - private static Vector2? FindReachablePoint( - VisualElement element, Rect bottomLeftBounds, float gameViewHeight, VisitContext context) + private static IReadOnlyList GenerateTopLeftProbePoints(Rect topLeftBounds, float gameViewHeight) { + Rect bottomLeftBounds = ToBottomLeft(topLeftBounds, gameViewHeight); Vector2[] bottomLeftProbes = UIElementAnnotator.GenerateReachabilityProbePoints( bottomLeftBounds.xMin, bottomLeftBounds.yMin, bottomLeftBounds.xMax, bottomLeftBounds.yMax); - Vector2[] topLeftProbes = bottomLeftProbes + return bottomLeftProbes .Select(probe => new Vector2(probe.x, gameViewHeight - probe.y)) .ToArray(); - return UIToolkitReachabilityProbe.FindReachablePoint( - topLeftProbes, element, context.FrontPickers, context.OwnPicker); } - private static UIElementInfo CreateElementInfo( - VisualElement element, - string type, - UIDocument document, - PanelEntry entry, - Rect bottomLeftBounds, - Vector2 bottomLeftPoint) + private static Rect ToBottomLeft(Rect topLeftBounds, float gameViewHeight) { - VisualElement? parent = element.hierarchy.parent; - return new UIElementInfo - { - Name = GetDisplayName(element), - Path = GameObjectPathUtility.GetFullPath(document.gameObject) + PATH_SEPARATOR - + BuildElementChain(document.rootVisualElement, element), - Type = type, - Interaction = UIElementAnnotationStyling.GetInteractionForType(type), - UiSystem = AnnotatedUiSystem.UIToolkit, - SimX = bottomLeftPoint.x, - SimY = bottomLeftPoint.y, - BoundsMinX = bottomLeftBounds.xMin, - BoundsMinY = bottomLeftBounds.yMin, - BoundsMaxX = bottomLeftBounds.xMax, - BoundsMaxY = bottomLeftBounds.yMax, - SortingOrder = Mathf.FloorToInt(entry.SortingOrder), - SiblingIndex = parent == null ? 0 : parent.hierarchy.IndexOf(element) - }; - } - - // Lists the elements from just below the document root down to the element, so the root itself, which - // has no name of its own, is left out. - private static string BuildElementChain(VisualElement root, VisualElement element) - { - List segments = new List(); - VisualElement? current = element; - while (current != null && current != root) - { - segments.Add(GetDisplayName(current)); - current = current.hierarchy.parent; - } - - segments.Reverse(); - return string.Join(CHAIN_SEPARATOR, segments); - } - - private static string GetDisplayName(VisualElement element) - { - return string.IsNullOrEmpty(element.name) ? element.GetType().Name : element.name; - } - - private static bool IsFiniteNonEmpty(Rect rect) - { - return float.IsFinite(rect.x) - && float.IsFinite(rect.y) - && float.IsFinite(rect.width) - && float.IsFinite(rect.height) - && rect.width > 0f - && rect.height > 0f; - } - - private sealed class PanelEntry - { - internal PanelEntry(IPanel panel, float sortingOrder, UIToolkitScreenMapping? mapping) - { - Panel = panel; - SortingOrder = sortingOrder; - Mapping = mapping; - } - - internal IPanel Panel { get; } - internal float SortingOrder { get; } - - // Null when the panel's controls cannot be placed on the screenshot; the panel still covers others. - internal UIToolkitScreenMapping? Mapping { get; } - internal List Documents { get; } = new List(); - } - - private sealed class VisitContext - { - internal VisitContext( - float gameViewHeight, - PanelEntry entry, - UIToolkitScreenMapping mapping, - List> frontPickers, - Func ownPicker, - List elements) - { - GameViewHeight = gameViewHeight; - Entry = entry; - Mapping = mapping; - FrontPickers = frontPickers; - OwnPicker = ownPicker; - Elements = elements; - } - - internal float GameViewHeight { get; } - internal PanelEntry Entry { get; } - internal UIToolkitScreenMapping Mapping { get; } - internal List> FrontPickers { get; } - internal Func OwnPicker { get; } - internal List Elements { get; } + return Rect.MinMaxRect( + topLeftBounds.xMin, + gameViewHeight - topLeftBounds.yMax, + topLeftBounds.xMax, + gameViewHeight - topLeftBounds.yMin); } } } diff --git a/Packages/src/Editor/FirstPartyTools/Screenshot/UnityCLILoop.FirstPartyTools.Screenshot.Editor.asmdef b/Packages/src/Editor/FirstPartyTools/Screenshot/UnityCLILoop.FirstPartyTools.Screenshot.Editor.asmdef index aa8752700b..fcf49663a7 100644 --- a/Packages/src/Editor/FirstPartyTools/Screenshot/UnityCLILoop.FirstPartyTools.Screenshot.Editor.asmdef +++ b/Packages/src/Editor/FirstPartyTools/Screenshot/UnityCLILoop.FirstPartyTools.Screenshot.Editor.asmdef @@ -5,6 +5,7 @@ "GUID:214998e563c124e8a88199b2dd1f522d", "GUID:5079a8d3a72924a81aa1cbc25f65ed1b", "GUID:a2d87883023de4cf59b7d1962d5dd8aa", + "GUID:569bba870e5e64d0d94144057b927001", "GUID:fc3fd32eddbee40e39c2d76dc184957b", "GUID:96b8d4624fb74ea2849fed17e7ad69d3", "GUID:77087b1bb30a415e87a80cbf24e7430a",