From ddbd15bc87d6bac0184c43106e9bb72ea348c960 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 5 Nov 2026 22:07:07 -0800 Subject: [PATCH] Fix iOS native editor and content-window targeting --- apple/DriveXCTestRunner.swift | 43 ++++++++++++++++++++++++++++------- apple/README.md | 26 +++++++++++++++++---- docs/api-migrations.md | 43 +++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 13 deletions(-) diff --git a/apple/DriveXCTestRunner.swift b/apple/DriveXCTestRunner.swift index e6ca686..6622319 100644 --- a/apple/DriveXCTestRunner.swift +++ b/apple/DriveXCTestRunner.swift @@ -132,13 +132,19 @@ public final class DriveXCTestRunner { private func window(_ snapshot: DriveProbeSnapshot) throws -> XCUIElement { guard app.state == .runningForeground else { throw DriveFailure.rejected("app is not foreground") } - let windows = app.windows.allElementsBoundByIndex.filter { $0.exists && !$0.frame.isEmpty } + let identifiers = Set(snapshot.nativeIdentifiers.values + .filter { $0.explicit && !$0.identifier.isEmpty }.map(\.identifier)).sorted() + guard !identifiers.isEmpty else { + throw DriveFailure.rejected("content window requires an explicit LUI accessibility-identifier") + } + let windows = app.windows.containing(NSPredicate(format: "identifier IN %@", identifiers)) guard windows.count == 1 else { - throw DriveFailure.rejected("Drive iOS requires exactly one visible application window") + throw DriveFailure.rejected("Drive iOS requires exactly one content window containing explicit LUI identifiers") } - let window = windows[0] + let window = windows.element let frame = window.frame - guard abs(frame.width - snapshot.viewport[2]) <= 1, + guard window.exists, !frame.isEmpty, + abs(frame.width - snapshot.viewport[2]) <= 1, abs(frame.height - snapshot.viewport[3]) <= 1 else { throw DriveFailure.rejected("native window and probe viewport disagree; rotation/multiple scenes unsupported") } @@ -162,6 +168,7 @@ public final class DriveXCTestRunner { frames[id] = [.int(id)] + frame.dropFirst().map { .double($0) } } var nativeState: [DriveJSON] = [] + let kinds = nodeKinds(snapshot) for key in snapshot.nativeIdentifiers.keys.sorted() { try client.check(deadline: deadline) guard let id = Int(key), ids.contains(id), let info = snapshot.nativeIdentifiers[key] else { @@ -169,7 +176,7 @@ public final class DriveXCTestRunner { } // Retained SwiftUI geometry is not used for native target actions. frames.removeValue(forKey: id) - let query = nativeQuery(identifier: info.identifier) + let query = nativeQuery(identifier: info.identifier, kind: kinds[id]) let count = info.identifier.isEmpty ? 0 : query.count var state: [DriveJSON] = [.int(id), .int(count)] if count == 1 { @@ -235,7 +242,7 @@ public final class DriveXCTestRunner { guard snapshot.nativeIdentifiers.values.filter({ $0.identifier == info.identifier }).count == 1 else { throw DriveFailure.rejected("effective accessibility-identifier is not unique in LUI") } - let query = nativeQuery(identifier: info.identifier) + let query = nativeQuery(identifier: info.identifier, kind: nodeKinds(snapshot)[id]) guard query.count == 1 else { throw DriveFailure.rejected("node \(id) has no unique native accessibility element (hidden/custom controls are unsupported)") } @@ -246,8 +253,28 @@ public final class DriveXCTestRunner { return element } - private func nativeQuery(identifier: String) -> XCUIElementQuery { - app.descendants(matching: .any).matching(NSPredicate(format: "identifier == %@", identifier)) + private func nodeKinds(_ snapshot: DriveProbeSnapshot) -> [Int: String] { + var kinds: [Int: String] = [:] + for op in snapshot.ops where op["op"]?.string == "create-node" { + if let id = op["id"]?.integer, let kind = op["kind"]?.string { kinds[id] = kind } + } + return kinds + } + + private func nativeQuery(identifier: String, kind: String?) -> XCUIElementQuery { + let query = app.descendants(matching: .any) + .matching(NSPredicate(format: "identifier == %@", identifier)) + let types: [XCUIElement.ElementType] + switch kind { + case "text-field", "input": types = [.textField] + case "secure-field": types = [.secureTextField] + case "search-field": types = [.searchField, .textField] + case "textarea": types = [.textView, .textField] + default: return query + } + // SwiftUI can propagate an editor identifier to auxiliary non-editor elements. + // Count all matching editors, including disabled/hidden ones, before target validation. + return query.matching(NSPredicate(format: "elementType IN %@", types.map { NSNumber(value: $0.rawValue) })) } private func coordinate(_ point: CGPoint, window: XCUIElement) throws -> XCUICoordinate { diff --git a/apple/README.md b/apple/README.md index 0bfc1e1..7c51bec 100644 --- a/apple/README.md +++ b/apple/README.md @@ -108,9 +108,17 @@ deadline longer than the runner's. Port collisions fail rather than binding a different interface. Project generation creates no simulator or app under test; Xcode build-for-testing creates the UI-test bundle/runner. Physical devices need explicit secure port forwarding for a Mac-hosted Drive client; automatic device transport is not -implemented. Multiple scenes/windows and rotation during an action are rejected -when native/probe viewport sizes disagree; use a fixed-orientation single-window -test fixture. +implemented. The runner requires exactly one content window containing at least +one explicit nonempty LUI identifier from the current probe snapshot (public +XCTest `containing` predicate). Zero or multiple anchored windows reject before +input; there is no window-order, hittability or private-class filtering. Keyboard +and text-service windows without those identifiers may coexist with the content +window. Unanchored/empty native trees are unsupported, even for screenshots or +settle; tag a real LUI control/content node such as the count label. Windows not +represented by the snapshot's identifiers cannot be recognized as independent +content. Multiple independent scenes and rotation remain unsupported; use one +fixed-orientation app scene. The foreground check and selected content window's +probe-viewport size check remain mandatory. ## Supported actions and limits @@ -119,10 +127,18 @@ The runner advertises exactly `click`, `double-click`, `long-click`, `tap`, - Target actions require an explicit, nonempty, stable LUI `accessibility-identifier`, unique among effective LUI identifiers and native - accessibility elements. The runner matches the **native identifier property**, + accessibility controls. Standard editor kinds additionally constrain the native + type: `text-field`/`input` → TextField, `secure-field` → SecureTextField, + `search-field` → SearchField/TextField, `textarea` → TextView/TextField. + SwiftUI's auxiliary non-editor elements may inherit the same identifier and + are excluded only for these editor kinds. Exactly one matching editor is + required even if other matching editors are hidden/disabled; enabled/hittable + validation happens after counting. Frame publication uses the same query. + Other kinds still require uniqueness across all native types. + The runner matches the **native identifier property**, never a label fallback, and freshly checks existence, enabled, and hittable. Untagged controls, inaccessible custom canvas controls, hidden/offscreen - controls, and identifier propagation creating multiple native matches fail + controls, and identifier propagation creating multiple eligible native matches fail explicitly. Extension authors must expose the same identifier on the actual native control. There is no coordinate fallback for target actions and no automatic scrolling to a target. diff --git a/docs/api-migrations.md b/docs/api-migrations.md index 8a58a43..9c1fd73 100644 --- a/docs/api-migrations.md +++ b/docs/api-migrations.md @@ -1,5 +1,48 @@ # API and wire migrations +## iOS content-window selection with the software keyboard + +The XCTest runner selects exactly one application window containing a native +descendant whose identifier equals an **explicit**, nonempty LUI identifier +from the current read-only snapshot. It uses public XCTest `containing` +predicates, not window order, private class names or hittability filtering. +The software keyboard and text-service windows can coexist with that content +window when they contain none of those identifiers. All anchored windows count: +zero or multiple matching content windows reject, even if one is hidden. +Foreground state and the selected window's probe-viewport size must still match. + +Empty/unanchored native trees are now explicitly unsupported, including for +settle/screenshot. Add a stable `accessibility-identifier` to a real LUI control +or content node (for example `counter` on the existing count label), then rebuild +the app/runner; no new wire fields or probe behavior are required. Only windows +represented by the snapshot's identifiers can be recognized as content; this +does not add support for multiple independent scenes or uninstrumented content +windows. Keep one fixed-orientation app scene. Control uniqueness, enabled and +hittable checks, window-local coordinates, input deadlines and poisoned-session +behavior are unchanged. + +## iOS native editor target selection + +The XCTest runner now resolves standard LUI editor nodes by their explicit +identifier **and native editor type**, for both frame publication/settling and +target actions. SwiftUI can propagate an editor identifier to a zero-size +auxiliary `Other` element as well as the real native editor; this no longer +makes the editor query ambiguous. + +`text-field`/`input` require `TextField`, `secure-field` requires +`SecureTextField`, `search-field` accepts `SearchField`/`TextField`, and +`textarea` accepts `TextView`/`TextField` (SwiftUI's vertical TextField). +Exactly one matching editor must exist, including hidden/disabled matches in +the count; multiple equivalent editors still reject before input. Effective +LUI identifier uniqueness and fresh enabled/hittable checks are unchanged. +Other node kinds retain all-types native uniqueness. There is no first-match, +label, coordinate or semantic fallback, and no event/wire changes. + +No scenario migration is required: keep, for example, `type testid:editor "hello"` +with an explicit `accessibility-identifier="editor"` on the LUI text-field node. +Rebuild the UI-test runner. Native replacement still uses public XCTest +Select All/Delete/typing and verifies exact retained text before success. + ## Live transport deadlines and fatal polling errors Native actions now use **one deadline for sending and completion**, including