diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m index f6df76a93c..158510fa07 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerAppFixtures.m @@ -220,7 +220,7 @@ - (void)viewDidLoad { } @end -@interface AgentDeviceRunnerFixtureViewController () +@interface AgentDeviceRunnerFixtureViewController () // Held for the app's lifetime: a window nobody retained would be gone before the test queried it. @property(nonatomic, strong) UIWindow *secondWindow; @property(nonatomic, strong) UILabel *navigationBackStatus; @@ -259,6 +259,7 @@ @interface AgentDeviceRunnerFixtureViewController () 1 { + _ = try tapAt(app: app, x: x, y: y, context: tapContext) + let reached = try textInputTheTapReached(candidates: candidates, contentTranslation: contentTranslation) + return TextEntryTarget( + element: reached.read.element, + refreshPoint: textEntryRefreshPoint(frame: reached.read.frameOnScreen, fallbackTo: requestedPoint), + prefersFocusedElement: false, + contentTranslation: contentTranslation, + boundIdentity: reached.input.identity, + postFocusRead: reached.read + ) + } if reusingFocusedInput, let input, let identity = input.identity, identity.isDistinguishable, let snapshot = input.snapshot, snapshotHasKeyboardFocus(snapshot) { NSLog("%@current-input-reports-keyboard-focus polls=0", TextEntryFocusEvidence.logMarker) @@ -159,7 +179,6 @@ extension RunnerTests { #endif let keyboardVisibleBeforeTap = isKeyboardVisible(app: app) let target = input?.element - let requestedPoint = CGPoint(x: x, y: y) // `tapAt` takes a screen point: with no screen box, tap the caller's own point (#3899, #3060). let tapPoint = input?.frameOnScreen.flatMap { $0.isEmpty ? nil : CGPoint(x: $0.midX, y: $0.midY) } ?? requestedPoint _ = try tapAt(app: app, x: tapPoint.x, y: tapPoint.y, context: tapContext) @@ -196,6 +215,54 @@ extension RunnerTests { return focusedTextEntryTarget(element, requestedPoint: requestedPoint, contentTranslation: contentTranslation) } +#if os(iOS) + /// The one of `candidates`, the inputs found under the caller's point before a tap there, that the + /// tap reached, read again through its own handle with its identity unchanged. A sheet or popover + /// leaves the inputs it covers in the tree, and neither size nor `isHittable`, which tests an + /// input's own hit point, names the input on top at the caller's point. An input that takes + /// keyboard focus after the tap is the one it reached, even when that focus lands some polls + /// later. An input that already held focus keeps it whatever the tap hit, so it counts only when no + /// candidate took focus by the deadline, and only as the one candidate uncovered then. Without + /// either, this throws before any clear or typing is dispatched. + func textInputTheTapReached( + candidates: [ResolvedTextEntryInput], + contentTranslation: TextEntryContentTranslation + ) throws -> (input: TextInputAtPoint, read: ResolvedTextEntryInput) { + let heldFocusBeforeTap = candidates.map { candidate in + candidate.snapshot.map { snapshotHasKeyboardFocus($0) } == true + } + var stillHeld: (index: Int, input: TextInputAtPoint, read: ResolvedTextEntryInput)? + let deadline = Date().addingTimeInterval(TextEntryTiming.readinessTimeout) + var polls = 0 + repeat { + polls += 1 + stillHeld = nil + for (index, candidate) in candidates.enumerated() { + guard let identity = candidate.identity else { continue } + let input = TextInputAtPoint(element: candidate.element, identity: identity) + guard let read = textInputIfStillResolves(input, contentTranslation: contentTranslation), + let snapshot = read.snapshot, snapshotHasKeyboardFocus(snapshot) else { continue } + guard !heldFocusBeforeTap[index] else { + stillHeld = (index, input, read) + continue + } + NSLog("%@%@ polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls) + return (input, read) + } + sleepFor(TextEntryTiming.pollInterval) + } while Date() < deadline + if let stillHeld { + let uncovered = candidates.map { isUncoveredTextInput($0) } + if uncovered[stillHeld.index], uncovered.filter({ $0 }).count == 1 { + NSLog("%@%@ polls=%d overlapping=1 held=1", TextEntryFocusEvidence.logMarker, TextEntryFocusEvidence.inputReportsKeyboardFocusName, polls) + return (stillHeld.input, stillHeld.read) + } + } + NSLog("%@none polls=%d overlapping=1", TextEntryFocusEvidence.logMarker, polls) + throw TextEntryFocusNotConfirmed() + } +#endif + /// The target the focus step hands to text entry, with the read its refresh point came from. private func focusedTextEntryTarget( _ element: XCUIElement?, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift index adba1bd27c..f229bebe22 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift @@ -39,10 +39,10 @@ extension RunnerTests { } /// What the point's text-input lookup can say before a coordinate gesture is dispatched: the - /// input it found, no input at the point, or nothing. A lookup that cannot answer is neither of the - /// first two, and must not be read as one of them. + /// input it found, with every input the point lies on, no input at the point, or nothing. A lookup + /// that cannot answer is neither of the first two, and must not be read as one of them. enum CoordinateTextInputResolution { - case input(TextInputAtPoint) + case input(TextInputAtPoint, candidates: [ResolvedTextEntryInput]) case absent case unavailable } @@ -59,7 +59,7 @@ extension RunnerTests { switch try probeTextInputs(app: app, point: CGPoint(x: x, y: y), contentTranslation: contentTranslation) { case .matches(let inputs): guard let input = inputs.first, let identity = input.identity else { return .unavailable } - return .input(TextInputAtPoint(element: input.element, identity: identity)) + return .input(TextInputAtPoint(element: input.element, identity: identity), candidates: inputs) case .absent: return .absent case .unavailable: @@ -212,6 +212,15 @@ extension RunnerTests { return inputs.sorted { smallestElementFirst($0.attributes, $1.attributes) } } + /// Whether XCTest reports `input` hittable: on screen, and covered by no other element at its hit + /// point. A read that records an issue or raises counts as covered. + func isUncoveredTextInput(_ input: ResolvedTextEntryInput) -> Bool { + guard case .success(let hittable) = containingTextInputProbeIssues(fallback: false, { _ in input.element.isHittable }) else { + return false + } + return hittable + } + /// The command's one content-vector read: the resolved window's screen origin when it is placed off /// the screen origin (#3899). A read that cannot answer is `.unknown`, never `.none`. func liveContentTranslation(app: XCUIApplication) -> TextEntryContentTranslation { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift index b80ca2b311..aaf9af4cc6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift @@ -2,8 +2,23 @@ import XCTest import AgentDeviceSnapshotPresentation extension RunnerTests { + /// Runs a `type` command. A focus step that could not confirm its input typed nothing, so it + /// answers as `TEXT_INPUT_NOT_FOCUSED`. @MainActor func executeTypeCommand(activeApp: XCUIApplication, command: Command) throws -> Response { + do { + return try typeCommandResponse(activeApp: activeApp, command: command) + } catch is TextEntryFocusNotConfirmed { + return textEntryFailureResponse(.notFocused) + } + } + + private func textEntryFailureResponse(_ failure: TextEntryFailure) -> Response { + Response(ok: false, error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)) + } + + @MainActor + private func typeCommandResponse(activeApp: XCUIApplication, command: Command) throws -> Response { guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) } @@ -45,9 +60,11 @@ extension RunnerTests { // answer both refuse here, each under its own reason. let contentTranslation = liveContentTranslation(app: activeApp) let inputAtPoint: TextInputAtPoint + let inputsAtPoint: [ResolvedTextEntryInput] switch try coordinateTapTextInputIdentityAt(app: activeApp, x: x, y: y, contentTranslation: contentTranslation) { - case .input(let input): + case .input(let input, let candidates): inputAtPoint = input + inputsAtPoint = candidates case .absent: return noTextInputAtPointResponse() case .unavailable: @@ -56,13 +73,24 @@ extension RunnerTests { let context = treeContext switch try performSynthesizedTapGesture(app: activeApp, x: x, y: y, context: context) { case .performed: + // The synthesized clear and typing reach whichever input holds focus, so where the point + // lies on several inputs, the one the tap reached is bound before any keystroke. + // The read that confirmed focus is reused, as on the XCTest route. + let reached: (input: TextInputAtPoint, read: ResolvedTextEntryInput?) + if inputsAtPoint.count > 1 { + let confirmed = try textInputTheTapReached(candidates: inputsAtPoint, contentTranslation: contentTranslation) + reached = (confirmed.input, confirmed.read) + } else { + reached = (inputAtPoint, nil) + } resolvedCoordinateContext = context resolvedCoordinateTarget = TextEntryTarget( element: nil, refreshPoint: CGPoint(x: x, y: y), prefersFocusedElement: false, contentTranslation: contentTranslation, - inputAtRefreshPoint: inputAtPoint + inputAtRefreshPoint: reached.input, + postFocusRead: reached.read ) case .xctestFallback: break @@ -122,10 +150,7 @@ extension RunnerTests { commandId: command.commandId ) if let failure = textResult.failure { - return Response( - ok: false, - error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint) - ) + return textEntryFailureResponse(failure) } if textResult.verified == false { return Response( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift index 3e6666d6cd..d7e3511cc6 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+CoordinateTextEntryTests.swift @@ -7,7 +7,7 @@ extension RunnerTests { @MainActor func requireTextInputAtPoint(app: XCUIApplication, x: Double, y: Double) throws -> TextInputAtPoint { switch try coordinateTapTextInputIdentityAt(app: app, x: x, y: y, contentTranslation: liveContentTranslation(app: app)) { - case .input(let input): + case .input(let input, _): return input case .absent: return try XCTUnwrap( diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift index 9a5e630e9d..d751849a11 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextEntryFocusTests.swift @@ -153,6 +153,240 @@ extension RunnerTests { } } + @MainActor + func testFillWithKeyboardDownTypesIntoTheSheetInputNotTheOneItCovers() throws { + for delayMs in [0, 80] { + let (foreground, background) = try launchOverlappingSheet() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: delayMs) + } + } + + @MainActor + func testFillWithKeyboardUpTypesIntoTheSheetInputNotTheOneItCovers() throws { + for delayMs in [0, 80] { + let (foreground, background) = try launchOverlappingSheet() + defer { invalidateCachedTarget(reason: "unit_test_cleanup"); app.terminate() } + app.textFields["agent-device-overlap-sheet-neighbour"].tap() + try XCTSkipUnless( + app.keyboards.firstMatch.waitForFixtureExistence(timeout: appExistenceTimeout), + "software keyboard is down: this simulator cannot exercise the keyboard-visible fill branch" + ) + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: delayMs) + } + } + + /// The sheet input refuses focus, so the neighbour that held it keeps it. On either channel the + /// fill refuses before any clear or typing, which would otherwise reach that neighbour. + @MainActor + func testFillRefusesAndKeepsTheFocusedFieldWhenTheSheetInputUnderThePointTakesNoFocus() throws { + for (penalized, delayMs) in [(false, 0), (true, 0), (true, 80)] { + let context = "penalized=\(penalized) delayMs=\(delayMs)" + let (foreground, background) = try launchOverlappingSheet(extra: ["--agent-device-text-entry-overlapping-sheet-refuses-focus"]) + defer { tearDownSynthesizedReplacementField() } + let neighbour = app.textFields["agent-device-overlap-sheet-neighbour"] + neighbour.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: neighbour), context) + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(foreground, text: "sheetvalue", delayMs: delayMs) + + XCTAssertFalse(response.ok, context) + XCTAssertEqual(response.error?.code, "TEXT_INPUT_NOT_FOCUSED", context) + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(foreground.value as? String, "stale", context) + XCTAssertEqual(background.value as? String, "background", context) + XCTAssertEqual(neighbour.value as? String, "neighbour", context) + } + } + + /// The popover's field covers the fill's point but not the center of the smaller field under it, + /// so XCTest reports both hittable and size alone picks the field under the popover. On either + /// channel, with or without that field focused first, the fill types into the popover's field. + @MainActor + func testFillAtAPointAPopoverCoversTypesIntoThePopoverFieldNotTheFieldUnderIt() throws { + for penalized in [false, true] { + for focusUnderFirst in [false, true] { + let context = "penalized=\(penalized) focusUnderFirst=\(focusUnderFirst)" + let (popoverField, partlyCovered, point) = try launchOverlappingPopover() + defer { tearDownSynthesizedReplacementField() } + if focusUnderFirst { + partlyCovered.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: partlyCovered), context) + } + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(popoverField, text: "popovervalue", at: point) + + XCTAssertTrue(response.ok, "\(context): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(popoverField.value as? String, "popovervalue", context) + XCTAssertEqual(partlyCovered.value as? String, "background", context) + } + } + } + + /// The field under the popover holds focus and is the only input at the point XCTest reports + /// hittable, while the popover's field, which the tap at the point reaches, takes focus a beat + /// later. On either channel the fill waits for that focus and types into the popover's field, not + /// into the field that held focus when the tap was dispatched. + @MainActor + func testFillTypesIntoThePopoverFieldThatTakesFocusLateNotTheFieldThatHeldFocus() throws { + for penalized in [false, true] { + let context = "penalized=\(penalized)" + let (popoverField, partlyCovered, point) = try launchOverlappingPopover(coveredLateFocus: true) + defer { tearDownSynthesizedReplacementField() } + partlyCovered.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: partlyCovered), context) + if penalized { try penalizeOverlappingFixtureChannel() } + let failures = currentXCTestFailureCount() + + let response = try executeWitnessEntry(popoverField, text: "popovervalue", at: point) + + XCTAssertTrue(response.ok, "\(context): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), context) + XCTAssertEqual(popoverField.value as? String, "popovervalue", context) + XCTAssertEqual(partlyCovered.value as? String, "background", context) + } + } + + /// The sheet's field already holds focus, so the tap at its center moves none. On either channel, + /// once no input there has taken focus, the fill types into it as the only uncovered input there. + @MainActor + func testFillTypesIntoTheFocusedSheetInputWhenTheTapMovesNoFocus() throws { + for penalized in [false, true] { + let (foreground, background) = try launchOverlappingSheet() + defer { tearDownSynthesizedReplacementField() } + foreground.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: foreground), "penalized=\(penalized)") + if penalized { try penalizeOverlappingFixtureChannel() } + try assertOverlappingSheetFill(foreground: foreground, background: background, delayMs: 0) + } + } + + /// Private synthesis clears the popover's field, which the tap at the point reached, and is gone + /// before the text is posted. Another lookup and tap at that point would find both fields + /// hittable and the popover's field already focused, so it would refuse on the emptied field; the + /// fallback types into the bound field instead. + @MainActor + func testFillWhoseSynthesisStopsAfterTheClearTypesIntoTheBoundPopoverField() throws { + let (popoverField, partlyCovered, point) = try launchOverlappingPopover() + defer { tearDownSynthesizedReplacementField() } + popoverField.tap() + XCTAssertTrue(keyboardFocusConfirmed(app: app, element: popoverField)) + try penalizeOverlappingFixtureChannel() + let bound = try requireTextInputAtPoint(app: app, x: popoverField.frame.midX, y: popoverField.frame.midY) + let failures = currentXCTestFailureCount() + + let result = try typeTextReliably( + app: app, + target: TextEntryTarget( + element: nil, + refreshPoint: point, + prefersFocusedElement: false, + contentTranslation: .none, + inputAtRefreshPoint: bound + ), + text: "popovervalue", + delaySeconds: 0, + repairMode: .replacement, + xCTestUnavailable: true, + synthesizer: SynthesisGoneAfterTheClear() + ) + + XCTAssertNil(result.failure) + XCTAssertEqual(result.textEntryRoute, "xctest-element") + XCTAssertFalse(didRecordXCTestFailure(since: failures)) + XCTAssertEqual(popoverField.value as? String, "popovervalue") + XCTAssertEqual(partlyCovered.value as? String, "background") + } + + /// Clears through private synthesis, then answers the text post as unavailable. + private struct SynthesisGoneAfterTheClear: TextEntrySynthesizing { + func enterText(app: XCUIApplication, text: String, replacingExistingText: Bool) -> SynthesizedTextEntryAction { + guard replacingExistingText else { return .fallback } + return PrivateXCTestTextEntrySynthesizer().enterText(app: app, text: text, replacingExistingText: true) + } + } + + /// The overlapping-sheet fixture, with the shape its tests depend on checked first: the sheet + /// input's center also lies in the input the sheet covers, which is the smaller of the two and + /// which XCTest reports not hittable. + @MainActor + private func launchOverlappingSheet(extra: [String] = []) throws -> (foreground: XCUIElement, background: XCUIElement) { + let (foreground, background) = launchOverlappingInputs( + ["--agent-device-text-entry-overlapping-sheet"] + extra, + top: "agent-device-overlap-foreground-input", + under: "agent-device-overlap-background-input" + ) + let sheetFrame = foreground.frame + let coveredFrame = background.frame + XCTAssertTrue(coveredFrame.contains(CGPoint(x: sheetFrame.midX, y: sheetFrame.midY))) + XCTAssertLessThan(coveredFrame.width * coveredFrame.height, sheetFrame.width * sheetFrame.height) + XCTAssertFalse(background.isHittable) + return (foreground, background) + } + + /// The overlapping-popover fixture, with the shape its tests depend on checked first: a point in + /// both fields, the partly covered field the smaller, its center outside the popover's field, and + /// the partly covered field hittable. The popover's field is hittable too, unless + /// `coveredLateFocus` lays a label over it and makes it take focus late. + @MainActor + private func launchOverlappingPopover( + coveredLateFocus: Bool = false + ) throws -> (popoverField: XCUIElement, partlyCovered: XCUIElement, point: CGPoint) { + let (popoverField, partlyCovered) = launchOverlappingInputs( + ["--agent-device-text-entry-overlapping-popover"] + + (coveredLateFocus ? ["--agent-device-text-entry-overlapping-popover-covered-late-focus"] : []), + top: "agent-device-popover-input", + under: "agent-device-popover-partly-covered-input" + ) + let popoverFrame = popoverField.frame + let coveredFrame = partlyCovered.frame + let point = CGPoint(x: (popoverFrame.minX + coveredFrame.maxX) / 2, y: coveredFrame.midY) + XCTAssertTrue(popoverFrame.contains(point)) + XCTAssertTrue(coveredFrame.contains(point)) + XCTAssertFalse(popoverFrame.contains(CGPoint(x: coveredFrame.midX, y: coveredFrame.midY))) + XCTAssertLessThan(coveredFrame.width * coveredFrame.height, popoverFrame.width * popoverFrame.height) + XCTAssertTrue(partlyCovered.isHittable) + XCTAssertEqual(popoverField.isHittable, !coveredLateFocus) + return (popoverField, partlyCovered, point) + } + + @MainActor + private func launchOverlappingInputs(_ arguments: [String], top: String, under: String) -> (top: XCUIElement, under: XCUIElement) { + app.launchArguments = arguments + app.launch() + XCTAssertTrue(app.waitForFixtureExistence(timeout: appExistenceTimeout)) + let topField = app.textFields[top] + let underField = app.textFields[under] + XCTAssertTrue(topField.waitForFixtureExistence(timeout: appExistenceTimeout)) + XCTAssertTrue(underField.exists) + return (topField, underField) + } + + /// Penalizes the XCTest channel for the running fixture, which routes a coordinate fill through + /// synthesized focus and typing. + @MainActor + private func penalizeOverlappingFixtureChannel() throws { + mainOwned.app = app + mainOwned.bundleId = "com.callstack.agentdevice.runner" + mainOwned.processIdentifier = try XCTUnwrap(Self.processIdentifier(of: app)) + penalizeSnapshotXCTestChannel(bundleId: nil, reason: "test") + } + + @MainActor + private func assertOverlappingSheetFill(foreground: XCUIElement, background: XCUIElement, delayMs: Int) throws { + let failures = currentXCTestFailureCount() + let response = try executeWitnessEntry(foreground, text: "sheetvalue", delayMs: delayMs) + XCTAssertTrue(response.ok, "delayMs=\(delayMs): \(String(describing: response.error))") + XCTAssertFalse(didRecordXCTestFailure(since: failures), "delayMs=\(delayMs)") + XCTAssertEqual(foreground.value as? String, "sheetvalue", "delayMs=\(delayMs)") + XCTAssertEqual(background.value as? String, "background", "delayMs=\(delayMs)") + } + @MainActor private func launchTextEntryWitness(prefilled: Bool = false, extra: [String] = []) throws -> XCUIElement { app.launchArguments = ["--agent-device-text-entry-regression", "--agent-device-text-entry-witness"] @@ -166,10 +400,17 @@ extension RunnerTests { } @MainActor - private func executeWitnessEntry(_ field: XCUIElement, text: String, mode: String = "replace", prepared: Bool = false) throws -> Response { - let frame = field.frame + private func executeWitnessEntry( + _ field: XCUIElement, + text: String, + mode: String = "replace", + delayMs: Int = 0, + at point: CGPoint? = nil, + prepared: Bool = false + ) throws -> Response { + let aim = point ?? CGPoint(x: field.frame.midX, y: field.frame.midY) let command = try runnerCommandFixture( - #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(frame.midX),"y":\#(frame.midY)}"# + #"{"command":"type","text":"\#(text)","textEntryMode":"\#(mode)","x":\#(aim.x),"y":\#(aim.y),"delayMs":\#(delayMs)}"# ) if prepared { return try executeOnMainPrepared(command: command, activeApp: app) diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 90dac45f0a..af0743b6ec 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -518,7 +518,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — ### Text entry - `fill` replaces the field value. `type` appends to the focused field and accepts text only: use `fill @ref "text"` to target a field, or `press @ref` then `type "text"`. -- iOS, including Limrun, `fill` avoids another focus tap when the current command identifies the target input and reads its keyboard focus. For readable focused inputs, clearing tries select-all/delete and reads the result before moving the caret for delete-burst fallback. Newly focused inputs retain caret preparation. A masked input, such as a secure field, is not cleared when it shows nothing or a placeholder that is not a mask. Otherwise it gets deletes with read-back first, and the caret tap at the field's right edge, where a show/hide control often sits, runs only when the read-back shows neither an empty value nor a masked placeholder. If the deletes can't reach the text, for example after the caret was moved off the end, and that text shows as many bullets as the placeholder, the read-back can't tell it from empty, so the new text is typed at the caret, next to the old text. +- iOS, including Limrun, `fill` avoids another focus tap when the current command identifies the target input, reads its keyboard focus, and finds no other input under its point. For readable focused inputs, clearing tries select-all/delete and reads the result before moving the caret for delete-burst fallback. Newly focused inputs retain caret preparation. A masked input, such as a secure field, is not cleared when it shows nothing or a placeholder that is not a mask. Otherwise it gets deletes with read-back first, and the caret tap at the field's right edge, where a show/hide control often sits, runs only when the read-back shows neither an empty value nor a masked placeholder. If the deletes can't reach the text, for example after the caret was moved off the end, and that text shows as many bullets as the placeholder, the read-back can't tell it from empty, so the new text is typed at the caret, next to the old text. - Use plain `fill` or `type` first. Add `--delay-ms` only when a debounced or search-as-you-type field misses characters, or the app must receive each incremental update; delayed typing enters characters one at a time instead of using a clipboard-style fallback. - `fill --text-stdin` reads the text from stdin so a secret never appears in process arguments. Pipe it with `printf %s`. It reads at most 64 KiB, removes one trailing newline, and refuses a terminal, empty input, invalid UTF-8, or an extra text argument. Output shows the value as `${VAR}` with `--record-as ` and as `[REDACTED]` otherwise, including in error messages and details, and in the `requested`, `before`, and `after` read-back of an unconfirmed fill, which can hold a reformatted form of the value. With `--settle`, the diff line text and tail labels are replaced whole, so text around the value in the field is not shown either; refs and counts stay. While script recording is armed, pass `--record-as ` or `--no-record`; passing neither is an error. Batch steps don't accept it. - The privacy controls for typed input (`--text-stdin` and `--record-as`) keep the literal out of agent-device's own responses, diagnostics, and recorded scripts. They do not rewrite what the platform records: on iOS, XCTest writes its own diagnostic logs and its `.xcresult` bundle, and typed text can appear in both, on a simulator or a physical device. Treat those as sensitive, and use test-only credentials when they may be kept or shared. See [Security and trust](/docs/security-trust). @@ -526,7 +526,7 @@ A comma-separated ref plan — `press @e13,@e14` or the same form on `click` — Text-entry errors and what to do: -- `TEXT_INPUT_NOT_FOCUSED` (`type`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. +- `TEXT_INPUT_NOT_FOCUSED` (`type`, iOS `fill`): focus a visible text input and retry. If accessibility does not expose the input, focus it by coordinates. An iOS `fill` whose point lies on more than one input, as on a sheet or popover over a form, taps that point and waits for keyboard focus. It clears and types only into the input there that took keyboard focus from that tap. When none took it, an input there that already had focus is used only if it still has focus and is the only uncovered input there; otherwise the fill returns this code and types nothing. If the input you want already has focus, move focus off it, for example with `keyboard dismiss`, then retry. - `NO_TEXT_INPUT_AT_POINT` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner identified no text input under that point and refused before dispatching anything (`error.details.reason` is `no_text_input_at_point`, `dispatched: no`), so no field changed. A ref or selector fill aims at the centre of the rect it resolved to, so an outer container or a non-input node can land outside the real field: run `snapshot -i`, pick the input node itself, and aim inside its rect. - `TEXT_INPUT_PROBE_UNAVAILABLE` (iOS `fill` under a degraded XCTest channel — coordinate, ref, selector, and replay fills all aim at a point): the runner's input lookup could not answer, so it refused before dispatching anything (`error.details.reason` is `text_input_probe_unavailable`, `dispatched: no`). This does not mean the point has no input. Retry the fill, or run `snapshot -i` once its capture reads back. A ref or selector fill aims at the centre of its resolved rect. - `TEXT_INPUT_SYNTHESIS_UNAVAILABLE` (iOS `type "\n"` with the software keyboard hidden): show the software keyboard, then retry.