From 58cc1d0a49c1aa35fdeea0bb03639ea6f329d12f Mon Sep 17 00:00:00 2001 From: Roberto Bortolaso <257786197+miospotdevteam@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:57:54 +0100 Subject: [PATCH 1/5] fix(apple-runner): fill the input a covering sheet leaves reachable A sheet or overlay can cover a text field that stays in the accessibility tree. When a fill's point lies on both fields, the point lookup ordered inputs smallest first with no occlusion check, so it picked the covered field. The focus tap still reached the sheet's field and raised the keyboard, the keyboard counted as focus evidence (or, already up, skipped the check), and the clear, typing and read-back went to the covered field: XCTest reported that it had no keyboard focus, or the read-back mismatched. - The point lookup puts inputs XCTest reports hittable ahead of covered ones when more than one input answers the point. - When more than one input answers the point, the iOS focus step accepts no keyboard evidence: the tapped input must report keyboard focus with its identity unchanged, and the target binds that identity for the clear, the typing and the read-back. Without that report it types nothing and answers TEXT_INPUT_NOT_FOCUSED. - A point with a single input keeps its current focus path, so secure fields and moved or replaced inputs behave as before. Adds an overlapping-sheet fixture and runner tests for the keyboard hidden and already shown, plain and with an 80 ms delay, and a sheet field that refuses focus. --- .../AgentDeviceRunnerAppFixtures.m | 56 ++++++++++++- .../RunnerTests+TextEntryFocus.swift | 48 +++++++++++- .../RunnerTests+TextInputProbe.swift | 21 ++++- .../RunnerTests+TypeExecution.swift | 20 ++++- .../RunnerTests+TextEntryFocusTests.swift | 78 ++++++++++++++++++- website/docs/docs/commands.md | 2 +- 6 files changed, 215 insertions(+), 10 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index f6df76a93c..86b6504be2 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -220,7 +220,7 @@ - (void)viewDidLoad { } @end -@interface AgentDeviceRunnerFixtureViewController () +@interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @property(nonatomic, strong) UILabel *navigationBackStatus; @@ -659,6 +659,22 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) { static const CGFloat AgentDeviceTextEntryFieldHeight = 44; static const CGFloat AgentDeviceTextEntryNeighbourGap = 16; +static UITextField *AgentDeviceOverlappingSheetField(NSString *identifier, NSString *text) { + UITextField *field = [[UITextField alloc] init]; + field.accessibilityIdentifier = identifier; + field.text = text; + field.borderStyle = UITextBorderStyleRoundedRect; + field.translatesAutoresizingMaskIntoConstraints = NO; + return field; +} + +// Under `--agent-device-text-entry-overlapping-sheet-refuses-focus`, the sheet's field is the only +// field with this delegate, and it never takes focus. +- (BOOL)textFieldShouldBeginEditing:(UITextField *)textField { + (void)textField; + return NO; +} + - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { if (self.maskedDeliveryStatus != nil) { self.maskedDeliveryStatus.text = [textField.text hasSuffix:@"not-a-secret-123"] ? @"delivered" : @"pending"; @@ -1241,6 +1257,44 @@ - (void)viewDidLoad { } } + // A sheet over a form, which leaves the form's field in the accessibility tree. The field the sheet + // covers is smaller than the sheet's field and shares its center, so a fill's point names both, and + // ordering by size alone picks the covered one. The sheet's second field raises the keyboard without + // touching either of them. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet"]) { + UITextField *background = AgentDeviceOverlappingSheetField(@"agent-device-overlap-background-input", @"background"); + UITextField *foreground = AgentDeviceOverlappingSheetField(@"agent-device-overlap-foreground-input", @"stale"); + UITextField *neighbour = AgentDeviceOverlappingSheetField(@"agent-device-overlap-sheet-neighbour", @""); + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet-refuses-focus"]) { + foreground.delegate = self; + } + UIView *sheet = [[UIView alloc] init]; + sheet.backgroundColor = UIColor.systemBackgroundColor; + sheet.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:background]; + [self.view addSubview:sheet]; + [sheet addSubview:foreground]; + [sheet addSubview:neighbour]; + [NSLayoutConstraint activateConstraints:@[ + [background.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [background.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-96], + [background.widthAnchor constraintEqualToConstant:240], + [background.heightAnchor constraintEqualToConstant:34], + [sheet.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor], + [sheet.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor], + [sheet.topAnchor constraintEqualToAnchor:background.topAnchor constant:-40], + [sheet.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor], + [foreground.centerXAnchor constraintEqualToAnchor:background.centerXAnchor], + [foreground.centerYAnchor constraintEqualToAnchor:background.centerYAnchor], + [foreground.widthAnchor constraintEqualToConstant:300], + [foreground.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], + [neighbour.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [neighbour.topAnchor constraintEqualToAnchor:foreground.bottomAnchor constant:AgentDeviceTextEntryNeighbourGap], + [neighbour.widthAnchor constraintEqualToConstant:300], + [neighbour.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], + ]]; + } + // The shape a SpringBoard confirmation root carries on an older iOS: the window's own title // text plus host chrome whose accessibility label is not copy (the Home Screen page readout), // placed first so a walk over every descendant reads it before the title. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 8565a88c35..799cf329f6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -1,5 +1,8 @@ import XCTest +/// The focus step tapped, but the input it aimed at did not report focus, so nothing was typed. +struct TextEntryFocusNotConfirmed: Error {} + extension RunnerTests { /// Books the tap as the address for the next bare `type`. `element` is a text input the caller /// classified BEFORE dispatching, or nil when the tap hit nothing worth typing into; that is the whole @@ -141,7 +144,8 @@ extension RunnerTests { return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true, contentTranslation: contentTranslation) } - let input = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first + let candidates = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) + let input = candidates.first #if os(iOS) if reusingFocusedInput, let input, let identity = input.identity, identity.isDistinguishable, let snapshot = input.snapshot, snapshotHasKeyboardFocus(snapshot) { @@ -163,6 +167,11 @@ extension RunnerTests { // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint _ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y, context: tapContext) +#if os(iOS) + if candidates.count > 1, let input { + return try overlappingTextEntryTarget(app: app, input: input, requestedPoint: requestedPoint, contentTranslation: contentTranslation) + } +#endif // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. @@ -196,6 +205,43 @@ extension RunnerTests { return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } +#if os(iOS) + /// The target for `input` when other inputs answer the same point, as fields a sheet covers do. A + /// keyboard, already up or coming up, shows only that some input took focus, so `input` itself must + /// report keyboard focus, read through its own handle with its identity unchanged. The target binds + /// that identity for the clear, the typing, and the read-back. Without that report the focus step + /// refuses: no other input under the point stands in for the one the tap was aimed at. + private func overlappingTextEntryTarget( + app: XCUIApplication, + input: ResolvedTextEntryInput, + requestedPoint: CGPoint, + contentTranslation: TextEntryContentTranslation + ) throws -> TextEntryTarget { + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var polls = 0 + repeat { + polls += 1 + if let read = ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation), + let identity = read.identity, input.identity == nil || identity == input.identity, + let snapshot = read.snapshot, + snapshotHasKeyboardFocus(snapshot) || keyboardFocusConfirmed(app: app, read: read) { + NSLog("%@%@ polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls) + return TextEntryTarget( + element: read.element, + refreshPoint: textEntryRefreshPoint(frame: read.frameOnScreen, fallbackTo: requestedPoint), + prefersFocusedElement: false, + contentTranslation: contentTranslation, + boundIdentity: identity, + postFocusRead: read + ) + } + sleepFor(TextEntryTiming.pollInterval) + } while Date() < deadline + NSLog("%@none polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, polls) + throw TextEntryFocusNotConfirmed() + } +#endif + /// The target the focus step hands to text entry, with the read its refresh point came from. private func focusedTextEntryTarget( _ element: XCUIElement?, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index adba1bd27c..529e302be0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -209,7 +209,26 @@ extension RunnerTests { } } guard !shouldStop() else { return [] } - return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } + let bySize = inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } + guard bySize.count > 1 else { return bySize } + // A sheet or overlay can cover an input that stays in the tree, so several inputs answer one + // point, and a covered one can be the smallest. A tap at the point reaches only an uncovered one. + var uncovered: [ResolvedTextEntryInput] = [] + var covered: [ResolvedTextEntryInput] = [] + for input in bySize { + if shouldStop() { return [] } + if isUncoveredTextInput(input) { uncovered.append(input) } else { covered.append(input) } + } + return uncovered + covered + } + + /// Whether XCTest reports `input` hittable: on screen, and covered by no other element at its hit + /// point. A read that records an issue or raises counts as covered. + private func isUncoveredTextInput(_ input: ResolvedTextEntryInput) -> Bool { + guard case .success(let hittable) = containingTextInputProbeIssues(fallback: false, { _ in input.element.isHittable }) else { + return false + } + return hittable } /// The command's one content-vector read: the resolved window's screen origin when it is placed off diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index b80ca2b311..24722e3287 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -2,8 +2,23 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { + /// Runs a `type` command. A focus step that could not confirm its input typed nothing, so it + /// answers as `TEXT_INPUT_NOT_FOCUSED`. @MainActor func executeTypeCommand(activeApp: XCUIApplication, command: Command) throws -> Response { + do { + return try typeCommandResponse(activeApp: activeApp, command: command) + } catch is TextEntryFocusNotConfirmed { + return textEntryFailureResponse(.notFocused) + } + } + + private func textEntryFailureResponse(_ failure: TextEntryFailure) -> Response { + Response(ok: false, error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)) + } + + @MainActor + private func typeCommandResponse(activeApp: XCUIApplication, command: Command) throws -> Response { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) } @@ -122,10 +137,7 @@ extension RunnerTests { commandId: command.commandId ) if let failure = textResult.failure { - return Response( - ok: false, - error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) - ) + return textEntryFailureResponse(failure) } if textResult.verified == false { return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift index 9a5e630e9d..9ce899f6fa 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -153,6 +153,74 @@ extension RunnerTests { } } + @MainActor + func testFillWithKeyboardDownTypesIntoTheSheetInputNotTheOneItCovers() throws { + for delayMs in [0, 80] { + let (foreground, background) = try launchOverlappingSheet() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: delayMs) + } + } + + @MainActor + func testFillWithKeyboardUpTypesIntoTheSheetInputNotTheOneItCovers() throws { + for delayMs in [0, 80] { + let (foreground, background) = try launchOverlappingSheet() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + app.textFields["agent-device-overlap-sheet-neighbour"].tap() + try XCTSkipUnless( + app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout), + "software keyboard is down: this simulator cannot exercise the keyboard-visible fill branch" + ) + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: delayMs) + } + } + + @MainActor + func testFillRefusesWhenTheSheetInputUnderThePointTakesNoFocus() throws { + let (foreground, background) = try launchOverlappingSheet(extra: ["--agent-device-text-entry-overlapping-sheet-refuses-focus"]) + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(foreground, text: "sheetvalue") + + XCTAssertFalse(response.ok) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertEqual(foreground.value as? String, "stale") + XCTAssertEqual(background.value as? String, "background") + } + + /// The overlapping-sheet fixture, with the shape its tests depend on checked first: the sheet + /// input's center also lies in the input the sheet covers, which is the smaller of the two and + /// which XCTest reports not hittable. + @MainActor + private func launchOverlappingSheet(extra: [String] = []) throws -> (foreground: XCUIElement, background: XCUIElement) { + app.launchArguments = ["--agent-device-text-entry-overlapping-sheet"] + extra + app.launch() + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let foreground = app.textFields["agent-device-overlap-foreground-input"] + let background = app.textFields["agent-device-overlap-background-input"] + XCTAssertTrue(foreground.waitForFixtureExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(background.exists) + let sheetFrame = foreground.frame + let coveredFrame = background.frame + XCTAssertTrue(coveredFrame.contains(CGPoint(x: sheetFrame.midX, y: sheetFrame.midY))) + XCTAssertLessThan(coveredFrame.width * coveredFrame.height, sheetFrame.width * sheetFrame.height) + XCTAssertFalse(background.isHittable) + return (foreground, background) + } + + @MainActor + private func assertOverlappingSheetFill(foreground: XCUIElement, background: XCUIElement, delayMs: Int) throws { + let failures = currentXCTestFailureCount() + let response = try executeWitnessEntry(foreground, text: "sheetvalue", delayMs: delayMs) + XCTAssertTrue(response.ok, "delayMs=\(delayMs): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), "delayMs=\(delayMs)") + XCTAssertEqual(foreground.value as? String, "sheetvalue", "delayMs=\(delayMs)") + XCTAssertEqual(background.value as? String, "background", "delayMs=\(delayMs)") + } + @MainActor private func launchTextEntryWitness(prefilled: Bool = false, extra: [String] = []) throws -> XCUIElement { app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-witness"] @@ -166,10 +234,16 @@ extension RunnerTests { } @MainActor - private func executeWitnessEntry(_ field: XCUIElement, text: String, mode: String = "replace", prepared: Bool = false) throws -> Response { + private func executeWitnessEntry( + _ field: XCUIElement, + text: String, + mode: String = "replace", + delayMs: Int = 0, + prepared: Bool = false + ) throws -> Response { let frame = field.frame let command = try runnerCommandFixture( - #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(frame.midX),"y":\#(frame.midY)}"# + #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(frame.midX),"y":\#(frame.midY),"delayMs":\#(delayMs)}"# ) if prepared { return try executeOnMainPrepared(command: command, activeApp: app) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 90dac45f0a..3d49230388 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -526,7 +526,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — Text-entry errors and what to do: -- `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. +- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet over a form, types into the uncovered one only after that input reports keyboard focus; otherwise it returns this code and types nothing. - `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point and refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`), so no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. - `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner's input lookup could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). This does not mean the point has no input. Retry the fill, or run `snapshot -i` once its capture reads back. A ref or selector fill aims at the centre of its resolved rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. From 1b9840b5f19b3f3791c558e89636de16b6c6dae8 Mon Sep 17 00:00:00 2001 From: Roberto Bortolaso <257786197+miospotdevteam@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:25:06 +0100 Subject: [PATCH 2/5] fix(apple-runner): type only into the input the tap at the point reached When a fill's point lies on more than one input, both routes chose the target without proving that a tap at that point reached it: the XCTest route ordered candidates by size and isHittable (an input's own hit point, not the caller's), and the synthesized route bound the probe's first candidate and sent its clear to whichever field held focus. Both now tap at the caller's point and accept only the input that took keyboard focus from it (or an already focused input that is the only uncovered one there), before any clear or typing; anything else is TEXT_INPUT_NOT_FOCUSED with nothing typed. New fixtures cover a popover that leaves the covered field's centre exposed and a sheet over a focused, populated neighbour. --- .../AgentDeviceRunnerAppFixtures.m | 23 +++- .../RunnerTests+TextEntryFocus.swift | 71 ++++++----- .../RunnerTests+TextInputProbe.swift | 10 +- .../RunnerTests+TypeExecution.swift | 11 +- ...RunnerTests+CoordinateTextEntryTests.swift | 2 +- .../RunnerTests+TextEntryFocusTests.swift | 120 +++++++++++++++--- website/docs/docs/commands.md | 4 +- 7 files changed, 180 insertions(+), 61 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 86b6504be2..1dc899cc27 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -1260,11 +1260,11 @@ - (void)viewDidLoad { // A sheet over a form, which leaves the form's field in the accessibility tree. The field the sheet // covers is smaller than the sheet's field and shares its center, so a fill's point names both, and // ordering by size alone picks the covered one. The sheet's second field raises the keyboard without - // touching either of them. + // touching either of them, and holds text that keystrokes sent to the wrong responder would change. if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet"]) { UITextField *background = AgentDeviceOverlappingSheetField(@"agent-device-overlap-background-input", @"background"); UITextField *foreground = AgentDeviceOverlappingSheetField(@"agent-device-overlap-foreground-input", @"stale"); - UITextField *neighbour = AgentDeviceOverlappingSheetField(@"agent-device-overlap-sheet-neighbour", @""); + UITextField *neighbour = AgentDeviceOverlappingSheetField(@"agent-device-overlap-sheet-neighbour", @"neighbour"); if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet-refuses-focus"]) { foreground.delegate = self; } @@ -1295,6 +1295,25 @@ - (void)viewDidLoad { ]]; } + // A popover's field over part of a smaller form field, leaving that field's center exposed: XCTest + // reports both hittable, and a point in the overlap lies on both. + if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-popover"]) { + UITextField *partlyCovered = AgentDeviceOverlappingSheetField(@"agent-device-popover-partly-covered-input", @"background"); + UITextField *popoverField = AgentDeviceOverlappingSheetField(@"agent-device-popover-input", @"stale"); + [self.view addSubview:partlyCovered]; + [self.view addSubview:popoverField]; + [NSLayoutConstraint activateConstraints:@[ + [partlyCovered.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [partlyCovered.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-96], + [partlyCovered.widthAnchor constraintEqualToConstant:160], + [partlyCovered.heightAnchor constraintEqualToConstant:34], + [popoverField.leadingAnchor constraintEqualToAnchor:partlyCovered.centerXAnchor constant:10], + [popoverField.centerYAnchor constraintEqualToAnchor:partlyCovered.centerYAnchor], + [popoverField.widthAnchor constraintEqualToConstant:180], + [popoverField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], + ]]; + } + // The shape a SpringBoard confirmation root carries on an older iOS: the window's own title // text plus host chrome whose accessibility label is not copy (the Home Screen page readout), // placed first so a walk over every descendant reads it before the title. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift index 799cf329f6..a0ecb1806b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift @@ -1,6 +1,7 @@ import XCTest -/// The focus step tapped, but the input it aimed at did not report focus, so nothing was typed. +/// The focus step tapped, but no input under the point was shown to take focus from that tap, so +/// nothing was typed. struct TextEntryFocusNotConfirmed: Error {} extension RunnerTests { @@ -146,7 +147,22 @@ extension RunnerTests { let candidates = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) let input = candidates.first + let requestedPoint = CGPoint(x: x, y: y) #if os(iOS) + // Focus the inputs already hold is not reused here: only a tap at the caller's point shows which + // of them that point reaches. + if candidates.count > 1 { + _ = try tapAt(app: app, x: x, y: y, context: tapContext) + let reached = try textInputTheTapReached(candidates: candidates, contentTranslation: contentTranslation) + return TextEntryTarget( + element: reached.read.element, + refreshPoint: textEntryRefreshPoint(frame: reached.read.frameOnScreen, fallbackTo: requestedPoint), + prefersFocusedElement: false, + contentTranslation: contentTranslation, + boundIdentity: reached.input.identity, + postFocusRead: reached.read + ) + } if reusingFocusedInput, let input, let identity = input.identity, identity.isDistinguishable, let snapshot = input.snapshot, snapshotHasKeyboardFocus(snapshot) { NSLog("%@current-input-reports-keyboard-focus polls=0", TextEntryFocusEvidence.logMarker) @@ -163,15 +179,9 @@ extension RunnerTests { #endif let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let target = input?.element - let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint _ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y, context: tapContext) -#if os(iOS) - if candidates.count > 1, let input { - return try overlappingTextEntryTarget(app: app, input: input, requestedPoint: requestedPoint, contentTranslation: contentTranslation) - } -#endif // A visible keyboard is not enough evidence for app.typeText, because focus may still // belong to a previous field. With a concrete target we type through XCUIElement.typeText, // so after tapping it the iOS readiness timeout cannot prove anything stronger. @@ -206,34 +216,37 @@ extension RunnerTests { } #if os(iOS) - /// The target for `input` when other inputs answer the same point, as fields a sheet covers do. A - /// keyboard, already up or coming up, shows only that some input took focus, so `input` itself must - /// report keyboard focus, read through its own handle with its identity unchanged. The target binds - /// that identity for the clear, the typing, and the read-back. Without that report the focus step - /// refuses: no other input under the point stands in for the one the tap was aimed at. - private func overlappingTextEntryTarget( - app: XCUIApplication, - input: ResolvedTextEntryInput, - requestedPoint: CGPoint, + /// The one of `candidates`, the inputs found under the caller's point before a tap there, that the + /// tap reached, read again through its own handle with its identity unchanged. A sheet or popover + /// leaves the inputs it covers in the tree, and neither size nor `isHittable`, which tests an + /// input's own hit point, names the input on top at the caller's point. An input that takes + /// keyboard focus after the tap is the one it reached. An input that already held focus keeps it + /// whatever the tap hit, so it counts only as the one candidate uncovered. Without either, this + /// throws before any clear or typing is dispatched. + func textInputTheTapReached( + candidates: [ResolvedTextEntryInput], contentTranslation: TextEntryContentTranslation - ) throws -> TextEntryTarget { + ) throws -> (input: TextInputAtPoint, read: ResolvedTextEntryInput) { + let heldFocusBeforeTap = candidates.map { candidate in + candidate.snapshot.map { snapshotHasKeyboardFocus($0) } == true + } + var uncoveredAfterTap: [Bool]? let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) var polls = 0 repeat { polls += 1 - if let read = ResolvedTextEntryInput(reading: input.element, contentTranslation: contentTranslation), - let identity = read.identity, input.identity == nil || identity == input.identity, - let snapshot = read.snapshot, - snapshotHasKeyboardFocus(snapshot) || keyboardFocusConfirmed(app: app, read: read) { + for (index, candidate) in candidates.enumerated() { + guard let identity = candidate.identity else { continue } + let input = TextInputAtPoint(element: candidate.element, identity: identity) + guard let read = textInputIfStillResolves(input, contentTranslation: contentTranslation), + let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) else { continue } + if heldFocusBeforeTap[index] { + let uncovered = uncoveredAfterTap ?? candidates.map { self.isUncoveredTextInput($0) } + uncoveredAfterTap = uncovered + guard uncovered[index], uncovered.filter({ $0 }).count == 1 else { continue } + } NSLog("%@%@ polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls) - return TextEntryTarget( - element: read.element, - refreshPoint: textEntryRefreshPoint(frame: read.frameOnScreen, fallbackTo: requestedPoint), - prefersFocusedElement: false, - contentTranslation: contentTranslation, - boundIdentity: identity, - postFocusRead: read - ) + return (input, read) } sleepFor(TextEntryTiming.pollInterval) } while Date() < deadline diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index 529e302be0..9154d665e7 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -39,10 +39,10 @@ extension RunnerTests { } /// What the point's text-input lookup can say before a coordinate gesture is dispatched: the - /// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the - /// first two, and must not be read as one of them. + /// input it found, with every input the point lies on, no input at the point, or nothing. A lookup + /// that cannot answer is neither of the first two, and must not be read as one of them. enum CoordinateTextInputResolution { - case input(TextInputAtPoint) + case input(TextInputAtPoint, candidates: [ResolvedTextEntryInput]) case absent case unavailable } @@ -59,7 +59,7 @@ extension RunnerTests { switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): guard let input = inputs.first, let identity = input.identity else { return .unavailable } - return .input(TextInputAtPoint(element: input.element, identity: identity)) + return .input(TextInputAtPoint(element: input.element, identity: identity), candidates: inputs) case .absent: return .absent case .unavailable: @@ -224,7 +224,7 @@ extension RunnerTests { /// Whether XCTest reports `input` hittable: on screen, and covered by no other element at its hit /// point. A read that records an issue or raises counts as covered. - private func isUncoveredTextInput(_ input: ResolvedTextEntryInput) -> Bool { + func isUncoveredTextInput(_ input: ResolvedTextEntryInput) -> Bool { guard case .success(let hittable) = containingTextInputProbeIssues(fallback: false, { _ in input.element.isHittable }) else { return false } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index 24722e3287..ce5c2f9d98 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -60,9 +60,11 @@ extension RunnerTests { // answer both refuse here, each under its own reason. let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint + let inputsAtPoint: [ResolvedTextEntryInput] switch try coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { - case .input(let input): + case .input(let input, let candidates): inputAtPoint = input + inputsAtPoint = candidates case .absent: return noTextInputAtPointResponse() case .unavailable: @@ -71,13 +73,18 @@ extension RunnerTests { let context = treeContext switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y, context: context) { case .performed: + // The synthesized clear and typing reach whichever input holds focus, so where the point + // lies on several inputs, the one the tap reached is bound before any keystroke. + let boundInput = try inputsAtPoint.count > 1 + ? textInputTheTapReached(candidates: inputsAtPoint, contentTranslation: contentTranslation).input + : inputAtPoint resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, contentTranslation: contentTranslation, - inputAtRefreshPoint: inputAtPoint + inputAtRefreshPoint: boundInput ) case .xctestFallback: break diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 3e6666d6cd..d7e3511cc6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -7,7 +7,7 @@ extension RunnerTests { @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { switch try coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { - case .input(let input): + case .input(let input, _): return input case .absent: return try XCTUnwrap( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift index 9ce899f6fa..75ab8a4015 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -176,19 +176,56 @@ extension RunnerTests { } } + /// The sheet input refuses focus, so the neighbour that held it keeps it. On either channel the + /// fill refuses before any clear or typing, which would otherwise reach that neighbour. @MainActor - func testFillRefusesWhenTheSheetInputUnderThePointTakesNoFocus() throws { - let (foreground, background) = try launchOverlappingSheet(extra: ["--agent-device-text-entry-overlapping-sheet-refuses-focus"]) - defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } - let failures = currentXCTestFailureCount() - - let response = try executeWitnessEntry(foreground, text: "sheetvalue") + func testFillRefusesAndKeepsTheFocusedFieldWhenTheSheetInputUnderThePointTakesNoFocus() throws { + for (penalized, delayMs) in [(false, 0), (true, 0), (true, 80)] { + let context = "penalized=\(penalized) delayMs=\(delayMs)" + let (foreground, background) = try launchOverlappingSheet(extra: ["--agent-device-text-entry-overlapping-sheet-refuses-focus"]) + defer { tearDownSynthesizedReplacementField() } + let neighbour = app.textFields["agent-device-overlap-sheet-neighbour"] + neighbour.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: neighbour), context) + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(foreground, text: "sheetvalue", delayMs: delayMs) + + XCTAssertFalse(response.ok, context) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED", context) + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(foreground.value as? String, "stale", context) + XCTAssertEqual(background.value as? String, "background", context) + XCTAssertEqual(neighbour.value as? String, "neighbour", context) + } + } - XCTAssertFalse(response.ok) - XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED") - XCTAssertFalse(didRecordXCTestFailure(since: failures)) - XCTAssertEqual(foreground.value as? String, "stale") - XCTAssertEqual(background.value as? String, "background") + /// The popover's field covers the fill's point but not the center of the smaller field under it, + /// so XCTest reports both hittable and size alone picks the field under the popover. On either + /// channel, with or without that field focused first, the fill types into the popover's field. + @MainActor + func testFillAtAPointAPopoverCoversTypesIntoThePopoverFieldNotTheFieldUnderIt() throws { + for penalized in [false, true] { + for focusUnderFirst in [false, true] { + let context = "penalized=\(penalized) focusUnderFirst=\(focusUnderFirst)" + let (popoverField, partlyCovered, point) = try launchOverlappingPopover() + defer { tearDownSynthesizedReplacementField() } + if focusUnderFirst { + partlyCovered.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: partlyCovered), context) + } + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(popoverField, text: "popovervalue", at: point) + + XCTAssertTrue(response.ok, "\(context): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(popoverField.value as? String, "popovervalue", context) + XCTAssertEqual(partlyCovered.value as? String, "background", context) + } + } } /// The overlapping-sheet fixture, with the shape its tests depend on checked first: the sheet @@ -196,13 +233,11 @@ extension RunnerTests { /// which XCTest reports not hittable. @MainActor private func launchOverlappingSheet(extra: [String] = []) throws -> (foreground: XCUIElement, background: XCUIElement) { - app.launchArguments = ["--agent-device-text-entry-overlapping-sheet"] + extra - app.launch() - XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) - let foreground = app.textFields["agent-device-overlap-foreground-input"] - let background = app.textFields["agent-device-overlap-background-input"] - XCTAssertTrue(foreground.waitForFixtureExistence(timeout: appExistenceTimeout)) - XCTAssertTrue(background.exists) + let (foreground, background) = launchOverlappingInputs( + ["--agent-device-text-entry-overlapping-sheet"] + extra, + top: "agent-device-overlap-foreground-input", + under: "agent-device-overlap-background-input" + ) let sheetFrame = foreground.frame let coveredFrame = background.frame XCTAssertTrue(coveredFrame.contains(CGPoint(x: sheetFrame.midX, y: sheetFrame.midY))) @@ -211,6 +246,50 @@ extension RunnerTests { return (foreground, background) } + /// The overlapping-popover fixture, with the shape its test depends on checked first: a point in + /// both fields, the partly covered field the smaller, its center outside the popover's field, and + /// both fields hittable. + @MainActor + private func launchOverlappingPopover() throws -> (popoverField: XCUIElement, partlyCovered: XCUIElement, point: CGPoint) { + let (popoverField, partlyCovered) = launchOverlappingInputs( + ["--agent-device-text-entry-overlapping-popover"], + top: "agent-device-popover-input", + under: "agent-device-popover-partly-covered-input" + ) + let popoverFrame = popoverField.frame + let coveredFrame = partlyCovered.frame + let point = CGPoint(x: (popoverFrame.minX + coveredFrame.maxX) / 2, y: coveredFrame.midY) + XCTAssertTrue(popoverFrame.contains(point)) + XCTAssertTrue(coveredFrame.contains(point)) + XCTAssertFalse(popoverFrame.contains(CGPoint(x: coveredFrame.midX, y: coveredFrame.midY))) + XCTAssertLessThan(coveredFrame.width * coveredFrame.height, popoverFrame.width * popoverFrame.height) + XCTAssertTrue(partlyCovered.isHittable) + XCTAssertTrue(popoverField.isHittable) + return (popoverField, partlyCovered, point) + } + + @MainActor + private func launchOverlappingInputs(_ arguments: [String], top: String, under: String) -> (top: XCUIElement, under: XCUIElement) { + app.launchArguments = arguments + app.launch() + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let topField = app.textFields[top] + let underField = app.textFields[under] + XCTAssertTrue(topField.waitForFixtureExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(underField.exists) + return (topField, underField) + } + + /// Penalizes the XCTest channel for the running fixture, which routes a coordinate fill through + /// synthesized focus and typing. + @MainActor + private func penalizeOverlappingFixtureChannel() throws { + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + } + @MainActor private func assertOverlappingSheetFill(foreground: XCUIElement, background: XCUIElement, delayMs: Int) throws { let failures = currentXCTestFailureCount() @@ -239,11 +318,12 @@ extension RunnerTests { text: String, mode: String = "replace", delayMs: Int = 0, + at point: CGPoint? = nil, prepared: Bool = false ) throws -> Response { - let frame = field.frame + let aim = point ?? CGPoint(x: field.frame.midX, y: field.frame.midY) let command = try runnerCommandFixture( - #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(frame.midX),"y":\#(frame.midY),"delayMs":\#(delayMs)}"# + #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(aim.x),"y":\#(aim.y),"delayMs":\#(delayMs)}"# ) if prepared { return try executeOnMainPrepared(command: command, activeApp: app) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 3d49230388..2ef74a4912 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -518,7 +518,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — ### Text entry - `fill` replaces the field value. `type` appends to the focused field and accepts text only: use `fill @ref "text"` to target a field, or `press @ref` then `type "text"`. -- iOS, including Limrun, `fill` avoids another focus tap when the current command identifies the target input and reads its keyboard focus. For readable focused inputs, clearing tries select-all/delete and reads the result before moving the caret for delete-burst fallback. Newly focused inputs retain caret preparation. A masked input, such as a secure field, is not cleared when it shows nothing or a placeholder that is not a mask. Otherwise it gets deletes with read-back first, and the caret tap at the field's right edge, where a show/hide control often sits, runs only when the read-back shows neither an empty value nor a masked placeholder. If the deletes can't reach the text, for example after the caret was moved off the end, and that text shows as many bullets as the placeholder, the read-back can't tell it from empty, so the new text is typed at the caret, next to the old text. +- iOS, including Limrun, `fill` avoids another focus tap when the current command identifies the target input, reads its keyboard focus, and finds no other input under its point. For readable focused inputs, clearing tries select-all/delete and reads the result before moving the caret for delete-burst fallback. Newly focused inputs retain caret preparation. A masked input, such as a secure field, is not cleared when it shows nothing or a placeholder that is not a mask. Otherwise it gets deletes with read-back first, and the caret tap at the field's right edge, where a show/hide control often sits, runs only when the read-back shows neither an empty value nor a masked placeholder. If the deletes can't reach the text, for example after the caret was moved off the end, and that text shows as many bullets as the placeholder, the read-back can't tell it from empty, so the new text is typed at the caret, next to the old text. - Use plain `fill` or `type` first. Add `--delay-ms` only when a debounced or search-as-you-type field misses characters, or the app must receive each incremental update; delayed typing enters characters one at a time instead of using a clipboard-style fallback. - `fill --text-stdin` reads the text from stdin so a secret never appears in process arguments. Pipe it with `printf %s`. It reads at most 64 KiB, removes one trailing newline, and refuses a terminal, empty input, invalid UTF-8, or an extra text argument. Output shows the value as `${VAR}` with `--record-as ` and as `[REDACTED]` otherwise, including in error messages and details, and in the `requested`, `before`, and `after` read-back of an unconfirmed fill, which can hold a reformatted form of the value. With `--settle`, the diff line text and tail labels are replaced whole, so text around the value in the field is not shown either; refs and counts stay. While script recording is armed, pass `--record-as ` or `--no-record`; passing neither is an error. Batch steps don't accept it. - The privacy controls for typed input (`--text-stdin` and `--record-as`) keep the literal out of agent-device's own responses, diagnostics, and recorded scripts. They do not rewrite what the platform records: on iOS, XCTest writes its own diagnostic logs and its `.xcresult` bundle, and typed text can appear in both, on a simulator or a physical device. Treat those as sensitive, and use test-only credentials when they may be kept or shared. See [Security and trust](/docs/security-trust). @@ -526,7 +526,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — Text-entry errors and what to do: -- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet over a form, types into the uncovered one only after that input reports keyboard focus; otherwise it returns this code and types nothing. +- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet or popover over a form, taps that point. It clears and types only into the input there that took keyboard focus from that tap, or into one that already had focus and is the only uncovered input there; otherwise it returns this code and types nothing. If the input you want already has focus, move focus off it, for example with `keyboard dismiss`, then retry. - `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point and refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`), so no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. - `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner's input lookup could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). This does not mean the point has no input. Retry the fill, or run `snapshot -i` once its capture reads back. A ref or selector fill aims at the centre of its resolved rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. From e64804c37c3348edfbc7dbe0b3c53d9c869a74a2 Mon Sep 17 00:00:00 2001 From: Roberto Bortolaso <257786197+miospotdevteam@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:56:31 +0100 Subject: [PATCH 3/5] fix(apple-runner): reuse the read that confirmed focus on the synthesized route The synthesized coordinate route discarded the read that confirmed the tapped input took focus and queried the input again before clearing. Pass it on as the XCTest route does. --- .../RunnerTests+TypeExecution.swift | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index ce5c2f9d98..aaf9af4cc6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -75,16 +75,22 @@ extension RunnerTests { case .performed: // The synthesized clear and typing reach whichever input holds focus, so where the point // lies on several inputs, the one the tap reached is bound before any keystroke. - let boundInput = try inputsAtPoint.count > 1 - ? textInputTheTapReached(candidates: inputsAtPoint, contentTranslation: contentTranslation).input - : inputAtPoint + // The read that confirmed focus is reused, as on the XCTest route. + let reached: (input: TextInputAtPoint, read: ResolvedTextEntryInput?) + if inputsAtPoint.count > 1 { + let confirmed = try textInputTheTapReached(candidates: inputsAtPoint, contentTranslation: contentTranslation) + reached = (confirmed.input, confirmed.read) + } else { + reached = (inputAtPoint, nil) + } resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, contentTranslation: contentTranslation, - inputAtRefreshPoint: boundInput + inputAtRefreshPoint: reached.input, + postFocusRead: reached.read ) case .xctestFallback: break From fb3d4abd0878485dcb7749fa3f83235a0c3153fe Mon Sep 17 00:00:00 2001 From: Roberto Bortolaso <257786197+miospotdevteam@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:21:13 +0100 Subject: [PATCH 4/5] fix(apple-runner): only the tap decides which overlapping input a fill reaches - The point lookup no longer puts hittable inputs first: it is back to smallest-first, so tap booking, read at a point and refresh resolution behave as before. - An input that takes focus from the tap wins, even when its focus lands late. An input that already had focus counts only when no other input took focus within the wait and it is then the one uncovered input there. - When synthesis stops after the clear, the fallback reuses the bound input while it still has keyboard focus instead of resolving the point again. --- .../AgentDeviceRunnerAppFixtures.m | 40 +++++++- .../RunnerTests+SynthesizedTextEntry.swift | 16 +++ .../RunnerTests+TextEntryFocus.swift | 24 +++-- .../RunnerTests+TextInputProbe.swift | 12 +-- .../RunnerTests+TextEntryFocusTests.swift | 97 ++++++++++++++++++- website/docs/docs/commands.md | 2 +- 6 files changed, 163 insertions(+), 28 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 1dc899cc27..803f5e629b 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -259,6 +259,7 @@ @interface AgentDeviceRunnerFixtureViewController () 1 else { return bySize } - // A sheet or overlay can cover an input that stays in the tree, so several inputs answer one - // point, and a covered one can be the smallest. A tap at the point reaches only an uncovered one. - var uncovered: [ResolvedTextEntryInput] = [] - var covered: [ResolvedTextEntryInput] = [] - for input in bySize { - if shouldStop() { return [] } - if isUncoveredTextInput(input) { uncovered.append(input) } else { covered.append(input) } - } - return uncovered + covered + return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } /// Whether XCTest reports `input` hittable: on screen, and covered by no other element at its hit diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift index 75ab8a4015..f234e53468 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -228,6 +228,89 @@ extension RunnerTests { } } + /// The field under the popover holds focus and is the only input at the point XCTest reports + /// hittable, while the popover's field, which the tap at the point reaches, takes focus a beat + /// later. On either channel the fill waits for that focus and types into the popover's field, not + /// into the field that held focus when the tap was dispatched. + @MainActor + func testFillTypesIntoThePopoverFieldThatTakesFocusLateNotTheFieldThatHeldFocus() throws { + for penalized in [false, true] { + let context = "penalized=\(penalized)" + let (popoverField, partlyCovered, point) = try launchOverlappingPopover(coveredLateFocus: true) + defer { tearDownSynthesizedReplacementField() } + partlyCovered.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: partlyCovered), context) + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(popoverField, text: "popovervalue", at: point) + + XCTAssertTrue(response.ok, "\(context): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(popoverField.value as? String, "popovervalue", context) + XCTAssertEqual(partlyCovered.value as? String, "background", context) + } + } + + /// The sheet's field already holds focus, so the tap at its center moves none. On either channel, + /// once no input there has taken focus, the fill types into it as the only uncovered input there. + @MainActor + func testFillTypesIntoTheFocusedSheetInputWhenTheTapMovesNoFocus() throws { + for penalized in [false, true] { + let (foreground, background) = try launchOverlappingSheet() + defer { tearDownSynthesizedReplacementField() } + foreground.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: foreground), "penalized=\(penalized)") + if penalized { try penalizeOverlappingFixtureChannel() } + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: 0) + } + } + + /// Private synthesis clears the popover's field, which the tap at the point reached, and is gone + /// before the text is posted. Another lookup and tap at that point would find both fields + /// hittable and the popover's field already focused, so it would refuse on the emptied field; the + /// fallback types into the bound field instead. + @MainActor + func testFillWhoseSynthesisStopsAfterTheClearTypesIntoTheBoundPopoverField() throws { + let (popoverField, partlyCovered, point) = try launchOverlappingPopover() + defer { tearDownSynthesizedReplacementField() } + popoverField.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: popoverField)) + try penalizeOverlappingFixtureChannel() + let bound = try requireTextInputAtPoint(app: app, x: popoverField.frame.midX, y: popoverField.frame.midY) + let failures = currentXCTestFailureCount() + + let result = try typeTextReliably( + app: app, + target: TextEntryTarget( + element: nil, + refreshPoint: point, + prefersFocusedElement: false, + contentTranslation: .none, + inputAtRefreshPoint: bound + ), + text: "popovervalue", + delaySeconds: 0, + repairMode: .replacement, + xCTestUnavailable: true, + synthesizer: SynthesisGoneAfterTheClear() + ) + + XCTAssertNil(result.failure) + XCTAssertEqual(result.textEntryRoute, "xctest-element") + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertEqual(popoverField.value as? String, "popovervalue") + XCTAssertEqual(partlyCovered.value as? String, "background") + } + + /// Clears through private synthesis, then answers the text post as unavailable. + private struct SynthesisGoneAfterTheClear: TextEntrySynthesizing { + func enterText(app: XCUIApplication, text: String, replacingExistingText: Bool) -> SynthesizedTextEntryAction { + guard replacingExistingText else { return .fallback } + return PrivateXCTestTextEntrySynthesizer().enterText(app: app, text: text, replacingExistingText: true) + } + } + /// The overlapping-sheet fixture, with the shape its tests depend on checked first: the sheet /// input's center also lies in the input the sheet covers, which is the smaller of the two and /// which XCTest reports not hittable. @@ -246,13 +329,17 @@ extension RunnerTests { return (foreground, background) } - /// The overlapping-popover fixture, with the shape its test depends on checked first: a point in + /// The overlapping-popover fixture, with the shape its tests depend on checked first: a point in /// both fields, the partly covered field the smaller, its center outside the popover's field, and - /// both fields hittable. + /// the partly covered field hittable. The popover's field is hittable too, unless + /// `coveredLateFocus` puts a banner over its center and makes it take focus late. @MainActor - private func launchOverlappingPopover() throws -> (popoverField: XCUIElement, partlyCovered: XCUIElement, point: CGPoint) { + private func launchOverlappingPopover( + coveredLateFocus: Bool = false + ) throws -> (popoverField: XCUIElement, partlyCovered: XCUIElement, point: CGPoint) { let (popoverField, partlyCovered) = launchOverlappingInputs( - ["--agent-device-text-entry-overlapping-popover"], + ["--agent-device-text-entry-overlapping-popover"] + + (coveredLateFocus ? ["--agent-device-text-entry-overlapping-popover-covered-late-focus"] : []), top: "agent-device-popover-input", under: "agent-device-popover-partly-covered-input" ) @@ -264,7 +351,7 @@ extension RunnerTests { XCTAssertFalse(popoverFrame.contains(CGPoint(x: coveredFrame.midX, y: coveredFrame.midY))) XCTAssertLessThan(coveredFrame.width * coveredFrame.height, popoverFrame.width * popoverFrame.height) XCTAssertTrue(partlyCovered.isHittable) - XCTAssertTrue(popoverField.isHittable) + XCTAssertEqual(popoverField.isHittable, !coveredLateFocus) return (popoverField, partlyCovered, point) } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 2ef74a4912..af0743b6ec 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -526,7 +526,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — Text-entry errors and what to do: -- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet or popover over a form, taps that point. It clears and types only into the input there that took keyboard focus from that tap, or into one that already had focus and is the only uncovered input there; otherwise it returns this code and types nothing. If the input you want already has focus, move focus off it, for example with `keyboard dismiss`, then retry. +- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet or popover over a form, taps that point and waits for keyboard focus. It clears and types only into the input there that took keyboard focus from that tap. When none took it, an input there that already had focus is used only if it still has focus and is the only uncovered input there; otherwise the fill returns this code and types nothing. If the input you want already has focus, move focus off it, for example with `keyboard dismiss`, then retry. - `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point and refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`), so no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. - `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner's input lookup could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). This does not mean the point has no input. Retry the fill, or run `snapshot -i` once its capture reads back. A ref or selector fill aims at the centre of its resolved rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry. From cb1f1af813805fc2b6284d152446f3e7d73cffb6 Mon Sep 17 00:00:00 2001 From: Roberto Bortolaso <257786197+miospotdevteam@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:50:13 +0100 Subject: [PATCH 5/5] test(apple-runner): the late-focus popover field reads covered to accessibility A plain view narrower than the field left its corners exposed and is not an accessibility element, so the field still read hittable. A labelled UILabel past every edge covers all the points the app reports from, while taking no touches. --- .../AgentDeviceRunnerAppFixtures.m | 24 +++++++++++-------- .../RunnerTests+TextEntryFocusTests.swift | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index 803f5e629b..158510fa07 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -1331,19 +1331,23 @@ - (void)viewDidLoad { [popoverField.widthAnchor constraintEqualToConstant:180], [popoverField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight], ]]; - // A banner over the popover field's center leaves the field under the popover the only one XCTest - // reports hittable, and the popover field takes focus late (see its delegate above). + // A hint label over the whole popover field, as a custom placeholder can be: a label takes no + // touches, so a tap there still reaches the popover field, but XCTest's hit test meets the label at + // every point it tries on that field (its center, its corners, just past its edges), so only the + // field under the popover reads hittable. A plain view would not do: it is no accessibility + // element, and that hit test looks through it. The popover field also takes focus late (see its + // delegate above). if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-popover-covered-late-focus"]) { popoverField.delegate = self; - UIView *banner = [[UIView alloc] init]; - banner.backgroundColor = UIColor.systemOrangeColor; - banner.translatesAutoresizingMaskIntoConstraints = NO; - [self.view addSubview:banner]; + UILabel *hint = [[UILabel alloc] init]; + hint.text = @"hint"; + hint.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:hint]; [NSLayoutConstraint activateConstraints:@[ - [banner.centerXAnchor constraintEqualToAnchor:popoverField.centerXAnchor], - [banner.centerYAnchor constraintEqualToAnchor:popoverField.centerYAnchor], - [banner.widthAnchor constraintEqualToConstant:40], - [banner.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight + 8], + [hint.leadingAnchor constraintEqualToAnchor:popoverField.leadingAnchor constant:-4], + [hint.trailingAnchor constraintEqualToAnchor:popoverField.trailingAnchor constant:4], + [hint.topAnchor constraintEqualToAnchor:popoverField.topAnchor constant:-4], + [hint.bottomAnchor constraintEqualToAnchor:popoverField.bottomAnchor constant:4], ]]; } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift index f234e53468..d751849a11 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -332,7 +332,7 @@ extension RunnerTests { /// The overlapping-popover fixture, with the shape its tests depend on checked first: a point in /// both fields, the partly covered field the smaller, its center outside the popover's field, and /// the partly covered field hittable. The popover's field is hittable too, unless - /// `coveredLateFocus` puts a banner over its center and makes it take focus late. + /// `coveredLateFocus` lays a label over it and makes it take focus late. @MainActor private func launchOverlappingPopover( coveredLateFocus: Bool = false