Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ - (void)viewDidLoad {
}
@end

@interface AgentDeviceRunnerFixtureViewController () <UNUserNotificationCenterDelegate>
@interface AgentDeviceRunnerFixtureViewController () <UNUserNotificationCenterDelegate, UITextFieldDelegate>
// Held for the app's lifetime: a window nobody retained would be gone before the test queried it.
@property(nonatomic, strong) UIWindow *secondWindow;
@property(nonatomic, strong) UILabel *navigationBackStatus;
Expand Down Expand Up @@ -259,6 +259,7 @@ @interface AgentDeviceRunnerFixtureViewController () <UNUserNotificationCenterDe
@property(nonatomic, weak) UITextField *keyboardBusyField;
@property(nonatomic, strong) UILabel *keyboardBusyAnswer;
@property(nonatomic, strong) NSTimer *keyboardBusyBackstop;
@property(nonatomic, assign) BOOL popoverLateFocusGranted;
@end

// A one-time-code field that announces how many digits it holds instead of the digits, the way
Expand Down Expand Up @@ -659,6 +660,40 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) {
static const CGFloat AgentDeviceTextEntryFieldHeight = 44;
static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;

static UITextField *AgentDeviceOverlappingSheetField(NSString *identifier, NSString *text) {
UITextField *field = [[UITextField alloc] init];
field.accessibilityIdentifier = identifier;
field.text = text;
field.borderStyle = UITextBorderStyleRoundedRect;
field.translatesAutoresizingMaskIntoConstraints = NO;
return field;
}

// Long enough that a read made right after the tap still finds focus where it was, and well inside
// the runner's 2 s wait for focus to land.
static const NSTimeInterval AgentDevicePopoverLateFocusSeconds = 1.0;

// The fields with this delegate: under `--agent-device-text-entry-overlapping-sheet-refuses-focus`,
// the sheet's field, which never takes focus; under
// `--agent-device-text-entry-overlapping-popover-covered-late-focus`, the popover's field, which takes
// focus a beat after the tap that asked for it, as a field whose focus goes through app code can.
- (BOOL)textFieldShouldBeginEditing:(UITextField *)textField {
if (![textField.accessibilityIdentifier isEqualToString:@"agent-device-popover-input"]) {
return NO;
}
if (self.popoverLateFocusGranted) {
self.popoverLateFocusGranted = NO;
return YES;
}
__weak AgentDeviceRunnerFixtureViewController *weakSelf = self;
__weak UITextField *weakField = textField;
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(AgentDevicePopoverLateFocusSeconds * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
weakSelf.popoverLateFocusGranted = YES;
[weakField becomeFirstResponder];
});
return NO;
}

- (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
if (self.maskedDeliveryStatus != nil) {
self.maskedDeliveryStatus.text = [textField.text hasSuffix:@"not-a-secret-123"] ? @"delivered" : @"pending";
Expand Down Expand Up @@ -1241,6 +1276,82 @@ - (void)viewDidLoad {
}
}

// A sheet over a form, which leaves the form's field in the accessibility tree. The field the sheet
// covers is smaller than the sheet's field and shares its center, so a fill's point names both, and
// ordering by size alone picks the covered one. The sheet's second field raises the keyboard without
// touching either of them, and holds text that keystrokes sent to the wrong responder would change.
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet"]) {
UITextField *background = AgentDeviceOverlappingSheetField(@"agent-device-overlap-background-input", @"background");
UITextField *foreground = AgentDeviceOverlappingSheetField(@"agent-device-overlap-foreground-input", @"stale");
UITextField *neighbour = AgentDeviceOverlappingSheetField(@"agent-device-overlap-sheet-neighbour", @"neighbour");
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-sheet-refuses-focus"]) {
foreground.delegate = self;
}
UIView *sheet = [[UIView alloc] init];
sheet.backgroundColor = UIColor.systemBackgroundColor;
sheet.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:background];
[self.view addSubview:sheet];
[sheet addSubview:foreground];
[sheet addSubview:neighbour];
[NSLayoutConstraint activateConstraints:@[
[background.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[background.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-96],
[background.widthAnchor constraintEqualToConstant:240],
[background.heightAnchor constraintEqualToConstant:34],
[sheet.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor],
[sheet.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor],
[sheet.topAnchor constraintEqualToAnchor:background.topAnchor constant:-40],
[sheet.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor],
[foreground.centerXAnchor constraintEqualToAnchor:background.centerXAnchor],
[foreground.centerYAnchor constraintEqualToAnchor:background.centerYAnchor],
[foreground.widthAnchor constraintEqualToConstant:300],
[foreground.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
[neighbour.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[neighbour.topAnchor constraintEqualToAnchor:foreground.bottomAnchor constant:AgentDeviceTextEntryNeighbourGap],
[neighbour.widthAnchor constraintEqualToConstant:300],
[neighbour.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
]];
}

// A popover's field over part of a smaller form field, leaving that field's center exposed: XCTest
// reports both hittable, and a point in the overlap lies on both.
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-popover"]) {
UITextField *partlyCovered = AgentDeviceOverlappingSheetField(@"agent-device-popover-partly-covered-input", @"background");
UITextField *popoverField = AgentDeviceOverlappingSheetField(@"agent-device-popover-input", @"stale");
[self.view addSubview:partlyCovered];
[self.view addSubview:popoverField];
[NSLayoutConstraint activateConstraints:@[
[partlyCovered.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[partlyCovered.bottomAnchor constraintEqualToAnchor:label.topAnchor constant:-96],
[partlyCovered.widthAnchor constraintEqualToConstant:160],
[partlyCovered.heightAnchor constraintEqualToConstant:34],
[popoverField.leadingAnchor constraintEqualToAnchor:partlyCovered.centerXAnchor constant:10],
[popoverField.centerYAnchor constraintEqualToAnchor:partlyCovered.centerYAnchor],
[popoverField.widthAnchor constraintEqualToConstant:180],
[popoverField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
]];
// A hint label over the whole popover field, as a custom placeholder can be: a label takes no
// touches, so a tap there still reaches the popover field, but XCTest's hit test meets the label at
// every point it tries on that field (its center, its corners, just past its edges), so only the
// field under the popover reads hittable. A plain view would not do: it is no accessibility
// element, and that hit test looks through it. The popover field also takes focus late (see its
// delegate above).
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-overlapping-popover-covered-late-focus"]) {
popoverField.delegate = self;
UILabel *hint = [[UILabel alloc] init];
hint.text = @"hint";
hint.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:hint];
[NSLayoutConstraint activateConstraints:@[
[hint.leadingAnchor constraintEqualToAnchor:popoverField.leadingAnchor constant:-4],
[hint.trailingAnchor constraintEqualToAnchor:popoverField.trailingAnchor constant:4],
[hint.topAnchor constraintEqualToAnchor:popoverField.topAnchor constant:-4],
[hint.bottomAnchor constraintEqualToAnchor:popoverField.bottomAnchor constant:4],
]];
}
}

// The shape a SpringBoard confirmation root carries on an older iOS: the window's own title
// text plus host chrome whose accessibility label is not copy (the Home Screen page readout),
// placed first so a walk over every descendant reads it before the title.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,22 @@ extension RunnerTests {
}
if synthesisAvailable.stoppedEarly {
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
// The bound input still holds focus, so it stays the target: another lookup and tap at the
// point, after the clear ran, could pick another input there or refuse on this one.
if let input = request.target.inputAtRefreshPoint,
let read = textInputIfStillResolves(input, contentTranslation: request.target.contentTranslation),
let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) {
return .fallback(
TextEntryTarget(
element: nil,
refreshPoint: request.target.refreshPoint,
prefersFocusedElement: false,
contentTranslation: request.target.contentTranslation,
inputAtRefreshPoint: input,
postFocusRead: read
)
)
}
guard let point = request.target.refreshPoint else { return .notApplicable }
return .fallback(
try focusTextInputForTextEntry(app: request.app, x: point.x, y: point.y)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import XCTest

/// The focus step tapped, but no input under the point was shown to take focus from that tap, so
/// nothing was typed.
struct TextEntryFocusNotConfirmed: Error {}

extension RunnerTests {
/// Books the tap as the address for the next bare `type`. `element` is a text input the caller
/// classified BEFORE dispatching, or nil when the tap hit nothing worth typing into; that is the whole
Expand Down Expand Up @@ -141,8 +145,24 @@ extension RunnerTests {
return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true, contentTranslation: contentTranslation)
}

let input = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation).first
let candidates = try textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation)
let input = candidates.first
let requestedPoint = CGPoint(x: x, y: y)
#if os(iOS)
// Focus the inputs already hold is not reused here: only a tap at the caller's point shows which
// of them that point reaches.
if candidates.count > 1 {
_ = try tapAt(app: app, x: x, y: y, context: tapContext)
let reached = try textInputTheTapReached(candidates: candidates, contentTranslation: contentTranslation)
return TextEntryTarget(
element: reached.read.element,
refreshPoint: textEntryRefreshPoint(frame: reached.read.frameOnScreen, fallbackTo: requestedPoint),
prefersFocusedElement: false,
contentTranslation: contentTranslation,
boundIdentity: reached.input.identity,
postFocusRead: reached.read
)
}
if reusingFocusedInput, let input, let identity = input.identity, identity.isDistinguishable,
let snapshot = input.snapshot, snapshotHasKeyboardFocus(snapshot) {
NSLog("%@current-input-reports-keyboard-focus polls=0", TextEntryFocusEvidence.logMarker)
Expand All @@ -159,7 +179,6 @@ extension RunnerTests {
#endif
let keyboardVisibleBeforeTap = isKeyboardVisible(app: app)
let target = input?.element
let requestedPoint = CGPoint(x: x, y: y)
// `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060).
let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint
_ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y, context: tapContext)
Expand Down Expand Up @@ -196,6 +215,54 @@ extension RunnerTests {
return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation)
}

#if os(iOS)
/// The one of `candidates`, the inputs found under the caller's point before a tap there, that the
/// tap reached, read again through its own handle with its identity unchanged. A sheet or popover
/// leaves the inputs it covers in the tree, and neither size nor `isHittable`, which tests an
/// input's own hit point, names the input on top at the caller's point. An input that takes
/// keyboard focus after the tap is the one it reached, even when that focus lands some polls
/// later. An input that already held focus keeps it whatever the tap hit, so it counts only when no
/// candidate took focus by the deadline, and only as the one candidate uncovered then. Without
/// either, this throws before any clear or typing is dispatched.
func textInputTheTapReached(
candidates: [ResolvedTextEntryInput],
contentTranslation: TextEntryContentTranslation
) throws -> (input: TextInputAtPoint, read: ResolvedTextEntryInput) {
let heldFocusBeforeTap = candidates.map { candidate in
candidate.snapshot.map { snapshotHasKeyboardFocus($0) } == true
}
var stillHeld: (index: Int, input: TextInputAtPoint, read: ResolvedTextEntryInput)?
let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout)
var polls = 0
repeat {
polls += 1
stillHeld = nil
for (index, candidate) in candidates.enumerated() {
guard let identity = candidate.identity else { continue }
let input = TextInputAtPoint(element: candidate.element, identity: identity)
guard let read = textInputIfStillResolves(input, contentTranslation: contentTranslation),
let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) else { continue }
guard !heldFocusBeforeTap[index] else {
stillHeld = (index, input, read)
continue
}
NSLog("%@%@ polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls)
return (input, read)
}
sleepFor(TextEntryTiming.pollInterval)
} while Date() < deadline
if let stillHeld {
let uncovered = candidates.map { isUncoveredTextInput($0) }
if uncovered[stillHeld.index], uncovered.filter({ $0 }).count == 1 {
NSLog("%@%@ polls=%d overlapping=1 held=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls)
return (stillHeld.input, stillHeld.read)
}
}
NSLog("%@none polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, polls)
throw TextEntryFocusNotConfirmed()
}
#endif

/// The target the focus step hands to text entry, with the read its refresh point came from.
private func focusedTextEntryTarget(
_ element: XCUIElement?,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,10 @@ extension RunnerTests {
}

/// What the point's text-input lookup can say before a coordinate gesture is dispatched: the
/// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the
/// first two, and must not be read as one of them.
/// input it found, with every input the point lies on, no input at the point, or nothing. A lookup
/// that cannot answer is neither of the first two, and must not be read as one of them.
enum CoordinateTextInputResolution {
case input(TextInputAtPoint)
case input(TextInputAtPoint, candidates: [ResolvedTextEntryInput])
case absent
case unavailable
}
Expand All @@ -59,7 +59,7 @@ extension RunnerTests {
switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) {
case .matches(let inputs):
guard let input = inputs.first, let identity = input.identity else { return .unavailable }
return .input(TextInputAtPoint(element: input.element, identity: identity))
return .input(TextInputAtPoint(element: input.element, identity: identity), candidates: inputs)
case .absent:
return .absent
case .unavailable:
Expand Down Expand Up @@ -212,6 +212,15 @@ extension RunnerTests {
return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) }
}

/// Whether XCTest reports `input` hittable: on screen, and covered by no other element at its hit
/// point. A read that records an issue or raises counts as covered.
func isUncoveredTextInput(_ input: ResolvedTextEntryInput) -> Bool {
guard case .success(let hittable) = containingTextInputProbeIssues(fallback: false, { _ in input.element.isHittable }) else {
return false
}
return hittable
}

/// The command's one content-vector read: the resolved window's screen origin when it is placed off
/// the screen origin (#3899). A read that cannot answer is `.unknown`, never `.none`.
func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation {
Expand Down
Loading