From 33c016f025a1fd454d2d0af3175f0dfeb2ad4c17 Mon Sep 17 00:00:00 2001 From: Junsoo Park Date: Fri, 25 Sep 2026 18:07:05 -0400 Subject: [PATCH 1/3] Record bounded T17 AX state when an identifier lookup times out --- .../BridgeVMProductE2E/T17Accessibility.swift | 2 +- .../T17MissingIdentifierDiagnostic.swift | 151 ++++++++++++++++++ .../T17MissingIdentifierDiagnosticTests.swift | 90 +++++++++++ scripts/refactor-budgets.tsv | 2 + 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 apps/macos/Sources/BridgeVMProductE2E/T17MissingIdentifierDiagnostic.swift create mode 100644 apps/macos/Tests/BridgeVMProductE2ETests/T17MissingIdentifierDiagnosticTests.swift diff --git a/apps/macos/Sources/BridgeVMProductE2E/T17Accessibility.swift b/apps/macos/Sources/BridgeVMProductE2E/T17Accessibility.swift index 60c65c66..64403f9d 100644 --- a/apps/macos/Sources/BridgeVMProductE2E/T17Accessibility.swift +++ b/apps/macos/Sources/BridgeVMProductE2E/T17Accessibility.swift @@ -130,7 +130,7 @@ final class T17Accessibility: T17UIControlling { RunLoop.current.run(until: Date().addingTimeInterval(0.1)) } while Date() < deadline let application = AXUIElementCreateApplication(pid) - throw T17Blocker(code: "ui-element-missing", detail: "required accessibility identifier was not found: \(identifier); windows=\((attribute(application, kAXWindowsAttribute as CFString) as? [AXUIElement]).map { String($0.count) } ?? "unanswered") timeout_s=\(timeout)") + throw T17MissingIdentifierDiagnostic.capture(application: application, pid: pid, identifier: identifier, timeout: timeout) } private func snapshotElement(_ identifier: String, role expectedRole: String?) throws -> AXUIElement? { diff --git a/apps/macos/Sources/BridgeVMProductE2E/T17MissingIdentifierDiagnostic.swift b/apps/macos/Sources/BridgeVMProductE2E/T17MissingIdentifierDiagnostic.swift new file mode 100644 index 00000000..ab831bed --- /dev/null +++ b/apps/macos/Sources/BridgeVMProductE2E/T17MissingIdentifierDiagnostic.swift @@ -0,0 +1,151 @@ +import AppKit +import ApplicationServices +import Darwin +import Foundation + +/// Failure-only metadata. Never reads AX titles, values, descriptions or paths. +enum T17MissingIdentifierDiagnostic { + struct Label: Hashable { + let role: String + let identifier: String + } + struct Inventory { + var nodes = 0 + var limited = false + var errors = 0 + var labels: [Label] = [] + } + struct Observation { + var pidProbe = "unknown" + var appPresent = false + var active: Bool? + var frontmost: Bool? + var windowsStatus = -1 + var windowsCount: Int? + var focusStatus = -1 + var focusPresent = false + var mainStatus = -1 + var mainPresent = false + var inventory = Inventory() + } + + private static let knownIDs: Set = [ + "bridgevm.dashboard.advanced", "bridgevm.windows.runtime.view", + "bridgevm.windows.install.view", "bridgevm.windows.runtime.start", + "bridgevm.windows.install.stage", "bridgevm.windows.install.failure", + ] + private static let knownRoles: Set = [ + "AXApplication", "AXWindow", "AXButton", "AXGroup", "AXScrollArea", + "AXStaticText", "AXDialog", "AXSheet", "AXUnknown", + ] + + static func capture(application: AXUIElement, pid: pid_t, identifier: String, + timeout: TimeInterval) -> T17Blocker { + var sample = Observation() + let probe = Darwin.kill(pid, 0) + sample.pidProbe = probe == 0 ? "present" : errno == ESRCH ? "missing" : errno == EPERM ? "denied" : "unknown" + let running = NSRunningApplication(processIdentifier: pid) + sample.appPresent = running != nil + sample.active = running?.isActive + sample.frontmost = NSWorkspace.shared.frontmostApplication.map { $0.processIdentifier == pid } + let timeoutStatus = AXUIElementSetMessagingTimeout(application, 0.25) + if timeoutStatus != .success { + sample.windowsStatus = Int(timeoutStatus.rawValue) + sample.focusStatus = Int(timeoutStatus.rawValue) + sample.mainStatus = Int(timeoutStatus.rawValue) + sample.inventory.errors = 1 + return failure(identifier: identifier, timeout: timeout, observation: sample) + } + let windows = query(application, kAXWindowsAttribute as CFString) + let windowNodes = windows.1 as? [AXUIElement] + sample.windowsStatus = Int(windows.0.rawValue) + sample.windowsCount = windowNodes?.count + let focus = query(application, kAXFocusedWindowAttribute as CFString) + sample.focusStatus = Int(focus.0.rawValue) + sample.focusPresent = focus.1.map { CFGetTypeID($0) == AXUIElementGetTypeID() } ?? false + let main = query(application, kAXMainWindowAttribute as CFString) + sample.mainStatus = Int(main.0.rawValue) + sample.mainPresent = main.1.map { CFGetTypeID($0) == AXUIElementGetTypeID() } ?? false + let roots = [application] + Array((windowNodes ?? []).prefix(8)) + let deadline = Date().addingTimeInterval(3) + sample.inventory = inventory(roots: roots, budget: { Date() < deadline }, + metadata: { node in + (try read(node, kAXRoleAttribute as CFString) as? String, + try read(node, kAXIdentifierAttribute as CFString) as? String) + }, related: { try read($0, kAXChildrenAttribute as CFString) as? [AXUIElement] ?? [] }, + same: { CFEqual($0, $1) }) + if (windowNodes?.count ?? 0) > 8 { sample.inventory.limited = true } + return failure(identifier: identifier, timeout: timeout, observation: sample) + } + + static func failure(identifier: String, timeout: TimeInterval, + observation: Observation) -> T17Blocker { + T17Blocker(code: "ui-element-missing", + detail: format(identifier: identifier, timeout: timeout, observation: observation)) + } + + static func inventory(roots: [Node], budget: () -> Bool, + metadata: (Node) throws -> (String?, String?), + related: (Node) throws -> [Node], + same: (Node, Node) -> Bool) -> Inventory { + let cap = 128 + var pending = Array(roots.prefix(cap)), seen: [Node] = [] + var cursor = 0 + var result = Inventory() + result.limited = roots.count > cap + var labels = Set