diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 136cf474f8..15d4a12c33 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -688,6 +688,20 @@ enum RetryableResponseFailure: Equatable { case targetAppUnavailable } +/// A control that dismisses the screen, such as a modal's Close button. The identifier is absent when +/// the control has none. `pressBy` names the attribute no other button shares, which the host turns +/// into the selector that presses it. +struct DismissControlPayload: Codable, Equatable { + enum PressBy: String, Codable { + case label + case identifier + } + + let label: String + let identifier: String? + let pressBy: PressBy +} + struct ErrorPayload: Codable { var code: String? let message: String @@ -695,11 +709,15 @@ struct ErrorPayload: Codable { /// Runner-internal: read by `shouldRetryResponse` and never encoded, so the host's decoding of /// the error is unchanged. var retryableFailure: RetryableResponseFailure? = nil + /// Only on an iOS `BACK_SWIPE_NO_CHANGE`: the dismiss controls in the top band the caller may press + /// instead. Absent when there are none. + var dismissControls: [DismissControlPayload]? = nil private enum CodingKeys: String, CodingKey { case code case message case hint + case dismissControls } static func targetAppUnavailable(bundleId: String?) -> ErrorPayload { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 1e4b4397fb..7376f426a1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -98,8 +98,9 @@ extension RunnerTests { /// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read /// from the live tree in this call, and confirms that the bar changed. Otherwise it is the - /// left-edge back swipe, verified by a change of the app's screen. No other button counts, - /// whatever its label. + /// left-edge back swipe, verified by a change of the app's screen. No other button is pressed, + /// whatever its label: a swipe that changed nothing names the dismiss controls the caller may press + /// instead (`topBandDismissControls`). @MainActor func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) @@ -137,8 +138,12 @@ extension RunnerTests { return pressed } do { - let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) + var outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) logSwipe(trace.logLine(outcome: outcome)) + if case .unconfirmed(var payload) = outcome, payload.code == RunnerWireErrorCode.backSwipeNoChange { + payload.dismissControls = topBandDismissControls(app: app, context: context) + outcome = .unconfirmed(payload) + } return outcome } catch { logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) @@ -508,12 +513,14 @@ extension RunnerTests { return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) } - /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, or the - /// final confirmation read. The read passes it to the test read hook. + /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, the + /// final confirmation read, or the dismiss control read after a swipe that changed nothing. The + /// read passes it to the test read hook. enum NavigationBarBackReadPhase { case lookup case confirmation case finalConfirmation + case dismissControlLookup } /// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung. @@ -659,6 +666,140 @@ extension RunnerTests { return state } + /// A dismiss control's label or identifier, compared as `selectorText`. These words only leave a + /// screen; "Done", "Save", and the like may commit, so they never count. + static let dismissControlWords: Set = ["back", "close", "cancel"] + + /// Text as the selector engine compares it (`normalizeText` in packages/selectors): trimmed, + /// lowercased, with each run of whitespace one space. A listed control's selector matches every + /// button whose label or identifier reads the same this way. + static func selectorText(_ text: String) -> String { + text.split(whereSeparator: \.isWhitespace).joined(separator: " ").lowercased() + } + + /// Whether `name` is a dismiss word, or an SF Symbols xmark name such as `xmark.circle.fill`. + static func isDismissControlName(_ name: String) -> Bool { + let word = selectorText(name) + return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.") + } + + /// Every button whose label or identifier contains a dismiss word, and every unlabeled one, whose + /// snapshot label can come from a child's text: a superset of the buttons a listed control's + /// selector matches. `selectorText` then decides on the label `press` resolves. + static let dismissControlPredicate: NSPredicate = { + let names = dismissControlWords.sorted() + ["xmark"] + let clauses = names.flatMap { _ in ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"] } + ["label == ''"] + return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] }) + }() + + /// One button the dismiss control read found: the label and identifier a snapshot node publishes for + /// it (`evaluateSnapshot`), which `press` resolves selectors against, and its snapshot frame in its + /// window's own space. + struct TopBandButton { + let label: String + let identifier: String + let frame: CGRect + } + + /// The button's label and identifier, the attributes a dismiss control's selector compares. + struct ButtonName { + let label: String + let identifier: String + } + + /// The app window's top band, where a bar or header holds its controls and content does not: + /// 22% of the window's height, held between 96 and 180 pt. + static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool { + guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window) else { + return false + } + let maxY = window.minY + min(max(window.height * 0.22, 96), 180) + return candidate.midY >= window.minY && candidate.midY <= maxY + } + + /// The roots a dismiss control is listed from: the window back swiped on when the context resolved + /// one, else every search root. A control in another window, such as an iPad's second scene, is + /// not on the screen back acted on. + static func dismissControlRoots(resolvedWindow: Root?, searchRoots: [Root]) -> [Root] { + resolvedWindow.map { [$0] } ?? searchRoots + } + + /// The dismiss controls a caller may press, in tree order: the buttons of the back window in the top + /// band of `window` whose label, or else identifier, is a dismiss name, pressed by that attribute. + /// `everyButton` holds the buttons of every window, which the selector `press` resolves can match: + /// a control whose attribute another of them shares is left out, because no selector presses it + /// alone. A system back button is never listed: back's own path presses or declines it. + /// `isHittable` is asked last, since each answer is an XCTest query. + static func dismissControls( + _ buttons: [TopBandButton], + everyButton: [ButtonName], + window: CGRect, + isHittable: (Int) -> Bool + ) -> [DismissControlPayload] { + buttons.indices.compactMap { index in + let button = buttons[index] + guard button.identifier != navigationBarBackButtonIdentifier, + isTopNavigationControlFrame(button.frame, in: window) else { return nil } + let pressBy: DismissControlPayload.PressBy + if isDismissControlName(button.label) { + pressBy = .label + } else if isDismissControlName(button.identifier) { + pressBy = .identifier + } else { + return nil + } + func attribute(_ name: ButtonName) -> String { selectorText(pressBy == .label ? name.label : name.identifier) } + let text = attribute(ButtonName(label: button.label, identifier: button.identifier)) + let matches = everyButton.filter { attribute($0) == text } + guard matches.count == 1, isHittable(index) else { return nil } + return DismissControlPayload( + label: button.label, identifier: button.identifier.isEmpty ? nil : button.identifier, pressBy: pressBy) + } + } + + /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe: every root's + /// buttons for the uniqueness check, and the back window's buttons, which are the same read when no + /// window resolved. It runs only on that + /// refusal, so a back that navigates pays nothing for it. Nil when there are none, or when the tree + /// is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list is advice, + /// and back refuses the same either way. + @MainActor + private func topBandDismissControls( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) -> [DismissControlPayload]? { + guard navigationBarBackButtonIsReadable(context: context) else { return nil } + let deadlineMs = currentUptimeMs() + Self.navigationBarBackReadBudgetMs + // Snapshot frames are in the window's own space, which a placed iPad window starts off the screen origin. + let window = CGRect(origin: .zero, size: context.geometry.windowOnScreen.rect.size) + let read = heldNavigationBarRead(.dismissControlLookup) { () -> [DismissControlPayload]? in + guard let searchRoots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { + return nil + } + @MainActor func buttons(under roots: [XCUIElement]) -> (elements: [XCUIElement], buttons: [TopBandButton])? { + let elements = roots.flatMap { + $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex + } + var buttons: [TopBandButton] = [] + for element in elements { + guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } + let node = evaluateSnapshot(snapshot) + buttons.append(TopBandButton(label: node.label, identifier: node.identifier, frame: snapshot.frame)) + } + return (elements, buttons) + } + guard let every = buttons(under: searchRoots) else { return nil } + let roots = Self.dismissControlRoots(resolvedWindow: context.resolvedWindow, searchRoots: searchRoots) + guard let listed = context.resolvedWindow == nil ? every : buttons(under: roots) else { return nil } + let everyButton = every.buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) } + return Self.dismissControls(listed.buttons, everyButton: everyButton, window: window) { + listed.elements[$0].isHittable + } + } + guard case .read(let controls) = read, !controls.isEmpty else { return nil } + return controls + } + @MainActor private func verifiedBackSwipe( app: XCUIApplication, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 3ffa22f4ab..981563d98c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -8,7 +8,8 @@ extension RunnerTests { code: "CODE", message: "message", hint: "hint", - retryableFailure: .targetAppUnavailable + retryableFailure: .targetAppUnavailable, + dismissControls: [DismissControlPayload(label: "Close", identifier: "close", pressBy: .label)] ) let encoded = try JSONEncoder().encode(payload) let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index d31c33fefc..3e0358c2e1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -339,6 +339,7 @@ extension RunnerTests { case .finalConfirmation: finalReads += 1 return nil + case .dismissControlLookup: return nil } } defer { navigationBarBackReadForTesting = nil } @@ -352,7 +353,7 @@ extension RunnerTests { } /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, - /// and the swipe that changes nothing reports so. + /// and the swipe that changes nothing reports so and names that button for the caller to press. @MainActor func testInAppBackNeverPressesABarButtonOnlyTitledBack() throws { let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--custom-back-item"]) @@ -362,6 +363,7 @@ extension RunnerTests { return XCTFail("a bar button titled Back must not count as the back button") } XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange) + XCTAssertEqual(payload.dismissControls, [DismissControlPayload(label: "Back", identifier: nil, pressBy: .label)]) XCTAssertEqual(status.label, "Navigation root") } @@ -902,6 +904,67 @@ extension RunnerTests { XCTAssertEqual(Self.backButtonCandidates([below, content], barFrames: [bar], windowHeight: 874), [0]) XCTAssertEqual(Self.backButtonCandidates([content], barFrames: [bar], windowHeight: 874), []) } + + /// The dismiss controls a no-change refusal names: hittable buttons in the window's top band whose + /// label or identifier is a dismiss name and no other button shares. An iPhone 17 Pro window: the band + /// ends at 180 pt. + func testDismissControlsListOnlyTopBandDismissButtons() { + let window = CGRect(x: 0, y: 0, width: 402, height: 874) + func button(_ label: String, identifier: String = "", y: CGFloat = 72) -> TopBandButton { + TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24)) + } + func controls(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set? = nil) -> [DismissControlPayload] { + let every = buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) } + others + return Self.dismissControls(buttons, everyButton: every, window: window) { hittable?.contains($0) ?? true } + } + func labels(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set? = nil) -> [String] { + controls(buttons, others: others, hittable: hittable).map(\.label) + } + XCTAssertEqual( + controls([button("Close")]), [DismissControlPayload(label: "Close", identifier: nil, pressBy: .label)], + "a lone Close in the band") + XCTAssertEqual(labels([button("Cancel"), button("Title"), button("close")]), ["Cancel", "close"], "both, in tree order") + XCTAssertEqual(labels([button("Done"), button("Save")]), [], "commit words never count") + XCTAssertEqual(labels([button("Close", y: 400)]), [], "a Close below the band") + XCTAssertEqual(labels([button("Close")], hittable: []), [], "an unhittable Close") + XCTAssertEqual(labels([button("Close account")]), [], "a longer label") + XCTAssertEqual( + labels([button("Cancel"), button(" cancel ", y: 600)]), [], + "a label another button reads the same as, to the selector engine, is pressed by no selector alone") + XCTAssertEqual( + controls([button("", identifier: "xmark.circle.fill")]), + [DismissControlPayload(label: "", identifier: "xmark.circle.fill", pressBy: .identifier)], + "an xmark icon identifier is pressed by it") + // The system back button is back's own target, pressed or declined by its own path, so it is never listed. + XCTAssertEqual(labels([button("Back", identifier: Self.navigationBarBackButtonIdentifier)]), []) + } + + /// The runner's text comparison is the selector engine's `normalizeText`: both are asserted on + /// contracts/fixtures/selector-text-normalization.json, so a listed selector is unique to `press`. + func testSelectorTextMatchesTheSelectorEngineNormalization() throws { + let cases = try XCTUnwrap(runnerContractFixture("selector-text-normalization.json") as? [[String: String]]) + XCTAssertFalse(cases.isEmpty) + for fixture in cases { + XCTAssertEqual(Self.selectorText(try XCTUnwrap(fixture["input"])), fixture["normalized"], fixture["name"] ?? "") + } + } + + /// Dismiss controls come from the window back swiped on; another window's buttons are not listed, + /// but a press resolves against every window, so one there that shares a label makes it ambiguous. + func testDismissControlsComeFromTheBackWindowAndAreUniqueAcrossEveryWindow() { + XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: "back window", searchRoots: ["app"]), ["back window"]) + XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: nil, searchRoots: ["left", "right"]), ["left", "right"]) + let window = CGRect(x: 0, y: 0, width: 834, height: 1194) + let close = TopBandButton(label: "Close", identifier: "", frame: CGRect(x: 780, y: 40, width: 30, height: 30)) + let backWindowClose = ButtonName(label: "Close", identifier: "") + XCTAssertEqual( + Self.dismissControls([close], everyButton: [backWindowClose], window: window) { _ in true }.map(\.label), ["Close"]) + XCTAssertEqual( + Self.dismissControls( + [close], everyButton: [backWindowClose, ButtonName(label: "close", identifier: "")], window: window + ) { _ in true }, + [], "a Close in a second window shares the selector") + } #endif /// A still screen settles before the budget runs out at any capture time: the settle loop's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 15aee46a14..08732576d6 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -212,6 +212,21 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | | `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | +A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…", "pressBy": "label" }`. +The `identifier` key is absent when the control has none. +`pressBy` is `label` when the label is the dismiss name, and `identifier` when only the identifier is. +The runner fills it after the swipe. It lists buttons of the window the swipe resolved (every search root when none resolved) and checks uniqueness against the buttons of every search root. +That read runs only on this refusal and only when the tree is readable. +Its snapshots start for up to 1 s. +An entry is a hittable button whose snapshot frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt. +Its label or identifier must be `back`, `close`, `cancel`, `xmark`, or an `xmark.` icon name, in any letter case. +A `BackButton` is never listed. +A button whose `pressBy` attribute another button shares, compared as the selector engine does (trimmed, lowercased, whitespace collapsed), is not listed, because a selector would match both. +The runner reads hittability only for the buttons that pass the other checks. +The runner never presses these controls. +With no entry, or when the read fails or hangs, the key is absent. +The daemon publishes the list as `details.dismissControls` and adds a selector to each entry: `role=button label="…"` or `role=button id="…"`. The `role=button` term limits the match to button elements, the set the runner checked. + The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact allowance, a 0.25 s margin, and the slowest capture seen while settling, about 0.55 s on a simulator. diff --git a/contracts/fixtures/selector-text-normalization.json b/contracts/fixtures/selector-text-normalization.json new file mode 100644 index 0000000000..6f40de8d4b --- /dev/null +++ b/contracts/fixtures/selector-text-normalization.json @@ -0,0 +1,19 @@ +[ + { + "name": "surrounding spaces and a tab are trimmed, case is folded", + "input": " Cancel\t", + "normalized": "cancel" + }, + { + "name": "a run of whitespace inside the text is one space", + "input": "Close\n\n Now", + "normalized": "close now" + }, + { "name": "no-break spaces count as whitespace", "input": " Back ", "normalized": "back" }, + { + "name": "an icon name keeps its dots", + "input": "XMARK.circle.fill", + "normalized": "xmark.circle.fill" + }, + { "name": "whitespace only is empty", "input": " \t\n", "normalized": "" } +] diff --git a/packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts b/packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts new file mode 100644 index 0000000000..121a6388f1 --- /dev/null +++ b/packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { buildRunnerResponseError } from '../runner-contract.ts'; + +const RUNNER_HINT = 'Press the visible back or close control by its ref from a fresh snapshot.'; + +function backRefusal(code: string, dismissControls?: unknown) { + return buildRunnerResponseError({ + ok: false, + error: { code, message: 'The back swipe changed nothing.', hint: RUNNER_HINT, dismissControls }, + }); +} + +test('a back swipe that changed nothing publishes its lone dismiss control and names it in the hint', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Close', pressBy: 'label' }]); + assert.equal(error.details?.reason, 'back_swipe_no_change'); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'role=button label="Close"', label: 'Close' }, + ]); + assert.equal( + error.details?.hint, + `Press 'role=button label="Close"' to dismiss this screen; back only pops the navigation stack.`, + ); +}); + +test('a dismiss control pressed by its identifier is selected by it', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [ + { label: 'xmark', identifier: 'sheet-close', pressBy: 'identifier' }, + ]); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'role=button id="sheet-close"', label: 'xmark', identifier: 'sheet-close' }, + ]); +}); + +test('two dismiss controls are both published and the hint points at the list', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [ + { label: 'Cancel', pressBy: 'label' }, + { label: 'Close', identifier: 'modal-close', pressBy: 'label' }, + ]); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'role=button label="Cancel"', label: 'Cancel' }, + { selector: 'role=button label="Close"', label: 'Close', identifier: 'modal-close' }, + ]); + assert.match(String(error.details?.hint), /one of details\.dismissControls/); +}); + +test('without dismiss controls the refusal keeps the runner hint and publishes no list', () => { + for (const controls of [ + undefined, + [], + [{ identifier: 7 }], + 'Close', + [{ label: 'Close' }], + [{ label: 'Close', pressBy: 'identifier' }], + ]) { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', controls); + assert.equal(error.details?.dismissControls, undefined); + assert.equal(error.details?.hint, RUNNER_HINT); + } +}); + +test('only the no-change refusal publishes dismiss controls', () => { + const error = backRefusal('BACK_SWIPE_UNVERIFIED', [{ label: 'Close', pressBy: 'label' }]); + assert.equal(error.details?.dismissControls, undefined); + assert.equal(error.details?.hint, RUNNER_HINT); +}); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 5bda230c30..f71f3f785f 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -555,10 +555,61 @@ export function classifyRunnerReportedError( export type RunnerResponsePayload = { ok?: unknown; - error?: { code?: unknown; message?: unknown; hint?: unknown }; + error?: { code?: unknown; message?: unknown; hint?: unknown; dismissControls?: unknown }; data?: unknown; }; +/** + * One entry of `details.dismissControls`: a control the caller may press to leave the screen an iOS + * `back` could not, such as a full-screen modal's Close button. `selector` presses it without a new + * snapshot. It is a selector, not a ref: back is a mutation, so it expires the session's ref frame + * before it swipes (ADR 0014), and the runner reads no snapshot tree to mint refs from. + */ +export type BackDismissControl = Readonly<{ selector: string; label: string; identifier?: string }>; + +/** + * The dismiss controls the runner names on a `BACK_SWIPE_NO_CHANGE` refusal, each with its selector. + * The runner's `pressBy` names the attribute no other button shares. Both forms carry `role=button`, + * which scopes the match to the buttons the runner checked, and resolves through the snapshot tree: + * `identifier` becomes `role=button id="…"`, and `label` becomes `role=button label="…"`. Other codes, and entries that are not + * `{ label, identifier?, pressBy }` objects, publish nothing. + */ +function readBackDismissControls( + runnerErrorCode: string | undefined, + error: RunnerResponsePayload['error'], +): BackDismissControl[] { + if (runnerErrorCode !== BACK_SWIPE_NO_CHANGE_RUNNER_CODE) return []; + const entries = error?.dismissControls; + if (!Array.isArray(entries)) return []; + return entries.flatMap((entry: unknown): BackDismissControl[] => { + if (typeof entry !== 'object' || entry === null) return []; + const { label, identifier, pressBy } = entry as { + label?: unknown; + identifier?: unknown; + pressBy?: unknown; + }; + if (typeof label !== 'string') return []; + const id = typeof identifier === 'string' && identifier.length > 0 ? identifier : undefined; + const selector = + pressBy === 'identifier' && id !== undefined + ? `role=button id=${JSON.stringify(id)}` + : pressBy === 'label' && label.length > 0 + ? `role=button label=${JSON.stringify(label)}` + : undefined; + if (selector === undefined) return []; + return [{ selector, label, ...(id === undefined ? {} : { identifier: id }) }]; + }); +} + +/** The refusal hint once dismiss controls are known; the runner's own hint otherwise. */ +function backDismissHint(controls: readonly BackDismissControl[]): string | undefined { + const [only] = controls; + if (only === undefined) return undefined; + return controls.length === 1 + ? `Press '${only.selector}' to dismiss this screen; back only pops the navigation stack.` + : 'Press one of details.dismissControls by its selector to dismiss this screen; back only pops the navigation stack.'; +} + /** * The one decoding of a runner response body (#2662). The envelope arrives at three readers — a * command's own response, the lifecycle journal a status probe reads back after the transport @@ -606,15 +657,17 @@ export function buildRunnerResponseError( typeof payload.error?.message === 'string' ? payload.error.message : undefined; const hint = typeof payload.error?.hint === 'string' ? payload.error.hint : undefined; const classification = classifyRunnerReportedError(runnerErrorCode); + const dismissControls = readBackDismissControls(runnerErrorCode, payload.error); return new AppError(classification.code, errorMessage ?? 'Runner error', { runner: payload, ...classification.details, + ...(dismissControls.length > 0 ? { dismissControls } : {}), xcodebuild: { exitCode: 1, stdout: '', stderr: '', }, - hint, + hint: backDismissHint(dismissControls) ?? hint, logPath, }); } diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 8da32ae664..2bf08dc1bd 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -304,7 +304,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: both modes only pop the navigation stack and never close a modal or sheet, whose own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS, back presses the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeeds only when the bar count, the back button count, or a back button title then changes; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its bar's title instead, which must hold still before the tap, and a bar that retitles itself can be misread as a pop. Otherwise it swipes from the left edge (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeeds only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the top-band Back, Close, Cancel, or xmark buttons (never Done or Save) as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector; back returns no refs, because it expires them. Reason back_swipe_unverified: the button was pressed but the bar did not change within 1 s or on one more read, or the swipe may or may not have navigated because the screen kept changing before it or could not be sampled after it. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/packages/selectors/src/internal/find-normalize-text.test.ts b/packages/selectors/src/internal/find-normalize-text.test.ts new file mode 100644 index 0000000000..b6f6cdc8bc --- /dev/null +++ b/packages/selectors/src/internal/find-normalize-text.test.ts @@ -0,0 +1,29 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { normalizeText } from './find.ts'; + +// Golden text normalization table: the Swift runner's `selectorText`, which decides whether iOS back +// lists a dismiss control's selector as unique, is asserted against the same JSON +// (UnitTests/RunnerTests+NavigationTests.swift). +type NormalizationCase = { name: string; input: string; normalized: string }; + +const TABLE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'selector-text-normalization.json', +); + +test('selector text normalization matches the shared table', () => { + const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as NormalizationCase[]; + assert.ok(cases.length > 0, 'the normalization table must not be empty'); + for (const fixture of cases) { + assert.equal(normalizeText(fixture.input), fixture.normalized, fixture.name); + } +}); diff --git a/packages/selectors/src/internal/match.test.ts b/packages/selectors/src/internal/match.test.ts index 18ce660854..09d8e7233c 100644 --- a/packages/selectors/src/internal/match.test.ts +++ b/packages/selectors/src/internal/match.test.ts @@ -280,3 +280,39 @@ test('appName selector matches nodes with appName field', () => { assert.ok(match3); assert.equal(match3.matches, 1); }); + +// iOS `back` lists a dismiss control with a role-scoped selector after the runner checked that no +// other button shares its label or identifier; the role term is what keeps non-buttons out. +test('role=button scopes a dismiss control selector to buttons, so same-text non-buttons stay out', () => { + const node = (ref: string, type: string, label: string, identifier = '') => ({ + ref, + index: Number(ref.slice(1)), + type, + kind: formatRole(type), + label, + identifier, + rect: { x: 366, y: 72, width: 24, height: 24 }, + enabled: true, + hittable: true, + }); + const nodes: SnapshotState['nodes'] = [ + node('n0', 'XCUIElementTypeStaticText', 'Close', 'modal-close'), + node('n1', 'XCUIElementTypeImage', 'close', 'modal-close'), + node('n2', 'XCUIElementTypeButton', 'Close', 'modal-close'), + ]; + for (const selector of ['role=button label="Close"', 'role=button id="modal-close"']) { + const resolved = resolveSelectorChain(nodes, parseSelectorChain(selector), { + platform: 'ios', + requireUnique: true, + }); + assert.equal(resolved?.node.ref, 'n2', selector); + } + // Without the role term the same label is ambiguous across the three element types. + assert.equal( + resolveSelectorChain(nodes, parseSelectorChain('label="Close"'), { + platform: 'ios', + requireUnique: true, + }), + null, + ); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 56f200bae9..c4a93be7ad 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -126,8 +126,9 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. macOS finds that control by the app's own back identifier or "Back" label; an app that exposes neither refuses with `error.details.reason: in_app_back_control_unavailable` under `COMMAND_FAILED` and `dispatched: no`, pressing nothing, so press the app's visible back control by its ref. +- iOS has no system back. On iOS, `back` only pops the navigation stack, by the back button or the edge swipe below. It does not dismiss a modal or a sheet: press that screen's own Close or Cancel control. Android's system back is different: it also closes dialogs and modal screens. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar, or, when none is, one floating below it, such as on the iPhone Duo), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the back buttons and navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. A back button with no title of its own, such as the Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap; when no bar has a title, or a title keeps changing, that button is not pressed and `back` swipes. If none changes, it reports `back_swipe_unverified` and does not swipe. For a back button that has its own title, a bar's title never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself, or a navigation bar that retitles itself under a back button with no title, can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). -- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. +- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `role=button label="…"` when the label is the dismiss word, and `role=button id="…"` when only the identifier is. Only controls in the app window that `back` swiped are listed. A control whose label or identifier another button in any of the app's windows shares (compared without letter case or extra spaces) is not listed, because its selector would match both. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. The field is absent when there is no control, when the app's tree cannot be read, or when the read fails, hangs, or runs past 1 s. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons (or, for a back button with no title, the bar's title) did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - Runner log `found=unprovable`: the back button by a navigation bar has no title and no navigation bar has one, so a pop could not be confirmed. The button is not pressed and `back` swipes instead. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device.