From 39b2c3ab71e97e5553bbbf948447ff23d7e1bcdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 11:56:40 +0100 Subject: [PATCH 1/5] feat(ios): name the dismiss controls when back changes nothing iOS back stays the navigation pop: the BackButton press, else the verified edge swipe. When the swipe changes nothing, such as on a full-screen modal, the refusal now lists the hittable Back, Close, Cancel or xmark buttons in the window's top band as details.dismissControls, each with a selector the caller can press without a new snapshot. The hint names a lone control. Back never presses them: Close or Cancel can discard a draft or end a flow. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Models.swift | 11 +++ .../RunnerTests+Navigation.swift | 95 ++++++++++++++++++- .../RunnerTests+NavigationTests.swift | 26 ++++- apple/runner/RUNNER_PROTOCOL.md | 12 +++ .../runner-back-dismiss-controls.test.ts | 56 +++++++++++ packages/apple-runner/src/runner-contract.ts | 47 ++++++++- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 3 +- 8 files changed, 242 insertions(+), 10 deletions(-) create mode 100644 packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 136cf474f8..eb29c9c972 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -688,6 +688,13 @@ enum RetryableResponseFailure: Equatable { case targetAppUnavailable } +/// A control that dismisses the screen, such as a modal's Close button. The identifier is absent when +/// the control has none. +struct DismissControlPayload: Codable, Equatable { + let label: String + let identifier: String? +} + struct ErrorPayload: Codable { var code: String? let message: String @@ -695,11 +702,15 @@ struct ErrorPayload: Codable { /// Runner-internal: read by `shouldRetryResponse` and never encoded, so the host's decoding of /// the error is unchanged. var retryableFailure: RetryableResponseFailure? = nil + /// Only on an iOS `BACK_SWIPE_NO_CHANGE`: the dismiss controls in the top band the caller may press + /// instead. Absent when there are none. + var dismissControls: [DismissControlPayload]? = nil private enum CodingKeys: String, CodingKey { case code case message case hint + case dismissControls } static func targetAppUnavailable(bundleId: String?) -> ErrorPayload { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 1e4b4397fb..1446e89f07 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -98,8 +98,9 @@ extension RunnerTests { /// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read /// from the live tree in this call, and confirms that the bar changed. Otherwise it is the - /// left-edge back swipe, verified by a change of the app's screen. No other button counts, - /// whatever its label. + /// left-edge back swipe, verified by a change of the app's screen. No other button is pressed, + /// whatever its label: a swipe that changed nothing names the dismiss controls the caller may press + /// instead (`topBandDismissControls`). @MainActor func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome { #if os(macOS) @@ -137,8 +138,12 @@ extension RunnerTests { return pressed } do { - let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) + var outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace) logSwipe(trace.logLine(outcome: outcome)) + if case .unconfirmed(var payload) = outcome, payload.code == RunnerWireErrorCode.backSwipeNoChange { + payload.dismissControls = topBandDismissControls(app: app, context: context) + outcome = .unconfirmed(payload) + } return outcome } catch { logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error)))) @@ -508,12 +513,14 @@ extension RunnerTests { return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId) } - /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, or the - /// final confirmation read. The read passes it to the test read hook. + /// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, the + /// final confirmation read, or the dismiss control read after a swipe that changed nothing. The + /// read passes it to the test read hook. enum NavigationBarBackReadPhase { case lookup case confirmation case finalConfirmation + case dismissControlLookup } /// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung. @@ -659,6 +666,84 @@ extension RunnerTests { return state } + /// A dismiss control's label or identifier, compared without case. These words only leave a screen; + /// "Done", "Save", and the like may commit, so they never count. + static let dismissControlWords: Set = ["back", "close", "cancel"] + + /// Whether `name` is a dismiss word, or an SF Symbols xmark name such as `xmark.circle.fill`. + static func isDismissControlName(_ name: String) -> Bool { + let word = name.lowercased() + return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.") + } + + /// Narrows the button query to the dismiss names; `isDismissControlName` decides. + static let dismissControlPredicate: NSPredicate = { + let names = dismissControlWords.sorted() + ["xmark"] + let clauses = names.flatMap { _ in ["label BEGINSWITH[c] %@", "identifier BEGINSWITH[c] %@"] } + return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] }) + }() + + /// One button the dismiss control read found, with its live frame in screen points. + struct TopBandButton { + let label: String + let identifier: String + let frame: CGRect + let isHittable: Bool + } + + /// The app window's top band, where a bar or header holds its controls and content does not: + /// 22% of the window's height, held between 96 and 180 pt. + static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool { + guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window) else { + return false + } + let maxY = window.minY + min(max(window.height * 0.22, 96), 180) + return candidate.midY >= window.minY && candidate.midY <= maxY + } + + /// The dismiss controls a caller may press: the hittable buttons in the top band of `window` whose + /// label or identifier is a dismiss name, in tree order. The system back button is never one, since + /// back itself presses it. + static func dismissControlCandidates(_ buttons: [TopBandButton], window: CGRect) -> [Int] { + buttons.indices.filter { index in + let button = buttons[index] + return button.isHittable && button.identifier != navigationBarBackButtonIdentifier + && isTopNavigationControlFrame(button.frame, in: window) + && (isDismissControlName(button.label) || isDismissControlName(button.identifier)) + } + } + + /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe. It runs + /// only on that refusal, so a back that navigates pays nothing for it. Nil when there are none, or + /// when the tree is not readable, a read fails, or a query hangs: the list is advice, and back + /// refuses the same either way. + @MainActor + private func topBandDismissControls( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) -> [DismissControlPayload]? { + guard navigationBarBackButtonIsReadable(context: context) else { return nil } + let deadlineMs = currentUptimeMs() + Self.navigationBarBackReadBudgetMs + let read = heldNavigationBarRead(.dismissControlLookup) { () -> [TopBandButton]? in + guard let roots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { return nil } + let elements = roots.flatMap { + $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex + } + var buttons: [TopBandButton] = [] + for element in elements { + guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } + buttons.append(TopBandButton( + label: snapshot.label, identifier: snapshot.identifier, frame: element.frame, isHittable: element.isHittable)) + } + return buttons + } + guard case .read(let buttons) = read else { return nil } + let controls = Self.dismissControlCandidates(buttons, window: context.geometry.windowOnScreen.rect).map { + DismissControlPayload(label: buttons[$0].label, identifier: buttons[$0].identifier.isEmpty ? nil : buttons[$0].identifier) + } + return controls.isEmpty ? nil : controls + } + @MainActor private func verifiedBackSwipe( app: XCUIApplication, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index d31c33fefc..c7a524d74d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -339,6 +339,7 @@ extension RunnerTests { case .finalConfirmation: finalReads += 1 return nil + case .dismissControlLookup: return nil } } defer { navigationBarBackReadForTesting = nil } @@ -352,7 +353,7 @@ extension RunnerTests { } /// A bar button titled "Back" is not the system back button, so back never presses it: it swipes, - /// and the swipe that changes nothing reports so. + /// and the swipe that changes nothing reports so and names that button for the caller to press. @MainActor func testInAppBackNeverPressesABarButtonOnlyTitledBack() throws { let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--custom-back-item"]) @@ -362,6 +363,7 @@ extension RunnerTests { return XCTFail("a bar button titled Back must not count as the back button") } XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange) + XCTAssertEqual(payload.dismissControls, [DismissControlPayload(label: "Back", identifier: nil)]) XCTAssertEqual(status.label, "Navigation root") } @@ -902,6 +904,28 @@ extension RunnerTests { XCTAssertEqual(Self.backButtonCandidates([below, content], barFrames: [bar], windowHeight: 874), [0]) XCTAssertEqual(Self.backButtonCandidates([content], barFrames: [bar], windowHeight: 874), []) } + + /// The dismiss controls a no-change refusal names: hittable buttons in the window's top band whose + /// label or identifier is a dismiss name. An iPhone 17 Pro window: the band ends at 180 pt. + func testDismissControlCandidatesListOnlyTopBandDismissButtons() { + let window = CGRect(x: 0, y: 0, width: 402, height: 874) + func button(_ label: String, identifier: String = "", y: CGFloat = 72, hittable: Bool = true) -> TopBandButton { + TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24), isHittable: hittable) + } + XCTAssertEqual(Self.dismissControlCandidates([button("Close")], window: window), [0], "a lone Close in the band") + XCTAssertEqual( + Self.dismissControlCandidates([button("Cancel"), button("Title"), button("close")], window: window), [0, 2], + "both candidates are listed, in tree order") + XCTAssertEqual(Self.dismissControlCandidates([button("Done"), button("Save")], window: window), [], "commit words never count") + XCTAssertEqual(Self.dismissControlCandidates([button("Close", y: 400)], window: window), [], "a Close below the band") + XCTAssertEqual(Self.dismissControlCandidates([button("Close", hittable: false)], window: window), []) + XCTAssertEqual( + Self.dismissControlCandidates([button("", identifier: "xmark.circle.fill"), button("Close account")], window: window), [0], + "an xmark icon name counts, a longer label does not") + // The system back button is back's own target, pressed by its own path, so it is never listed. + XCTAssertEqual( + Self.dismissControlCandidates([button("Back", identifier: Self.navigationBarBackButtonIdentifier)], window: window), []) + } #endif /// A still screen settles before the budget runs out at any capture time: the settle loop's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 15aee46a14..65b4be08d8 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -212,6 +212,18 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | | `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | +A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…" }`. +The `identifier` key is absent when the control has none. +The runner fills it after the swipe, from one read of the buttons in the app's windows. +That read runs only on this refusal and only when the tree is readable. +Its snapshots start for up to 1 s. +An entry is a hittable button whose live frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt. +Its label or identifier must be `back`, `close`, `cancel`, `xmark`, or an `xmark.` icon name, in any letter case. +A `BackButton` is never listed. +The runner never presses these controls. +With no entry, or when the read fails or hangs, the key is absent. +The daemon publishes the list as `details.dismissControls` and adds a selector to each entry. + The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact allowance, a 0.25 s margin, and the slowest capture seen while settling, about 0.55 s on a simulator. diff --git a/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts b/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts new file mode 100644 index 0000000000..acf4425c4c --- /dev/null +++ b/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { buildRunnerResponseError } from '../runner-contract.ts'; + +const RUNNER_HINT = 'Press the visible back or close control by its ref from a fresh snapshot.'; + +function backRefusal(code: string, dismissControls?: unknown) { + return buildRunnerResponseError({ + ok: false, + error: { code, message: 'The back swipe changed nothing.', hint: RUNNER_HINT, dismissControls }, + }); +} + +test('a back swipe that changed nothing publishes its lone dismiss control and names it in the hint', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Close' }]); + assert.equal(error.details?.reason, 'back_swipe_no_change'); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'role=button label="Close"', label: 'Close' }, + ]); + assert.equal( + error.details?.hint, + `Press 'role=button label="Close"' to dismiss this screen; back only pops the navigation stack.`, + ); +}); + +test('a dismiss control with an identifier is selected by it', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [ + { label: 'xmark', identifier: 'sheet-close' }, + ]); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'id="sheet-close"', label: 'xmark', identifier: 'sheet-close' }, + ]); +}); + +test('two dismiss controls are both published and the hint points at the list', () => { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Cancel' }, { label: 'Close' }]); + assert.deepEqual( + (error.details?.dismissControls as { label: string }[]).map((control) => control.label), + ['Cancel', 'Close'], + ); + assert.match(String(error.details?.hint), /one of details\.dismissControls/); +}); + +test('without dismiss controls the refusal keeps the runner hint and publishes no list', () => { + for (const controls of [undefined, [], [{ identifier: 7 }], 'Close']) { + const error = backRefusal('BACK_SWIPE_NO_CHANGE', controls); + assert.equal(error.details?.dismissControls, undefined); + assert.equal(error.details?.hint, RUNNER_HINT); + } +}); + +test('only the no-change refusal publishes dismiss controls', () => { + const error = backRefusal('BACK_SWIPE_UNVERIFIED', [{ label: 'Close' }]); + assert.equal(error.details?.dismissControls, undefined); + assert.equal(error.details?.hint, RUNNER_HINT); +}); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index 5bda230c30..c9d86a2391 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -555,10 +555,51 @@ export function classifyRunnerReportedError( export type RunnerResponsePayload = { ok?: unknown; - error?: { code?: unknown; message?: unknown; hint?: unknown }; + error?: { code?: unknown; message?: unknown; hint?: unknown; dismissControls?: unknown }; data?: unknown; }; +/** + * One entry of `details.dismissControls`: a control the caller may press to leave the screen an iOS + * `back` could not, such as a full-screen modal's Close button. `selector` presses it without a new + * snapshot. It is a selector, not a ref: back is a mutation, so it expires the session's ref frame + * before it swipes (ADR 0014), and the runner reads no snapshot tree to mint refs from. + */ +export type BackDismissControl = Readonly<{ selector: string; label: string; identifier?: string }>; + +/** + * The dismiss controls the runner names on a `BACK_SWIPE_NO_CHANGE` refusal, each with its selector: + * the identifier when the control has one, otherwise its button role and label. Other codes, and + * entries that are not `{ label, identifier? }` objects, publish nothing. + */ +function readBackDismissControls( + runnerErrorCode: string | undefined, + error: RunnerResponsePayload['error'], +): BackDismissControl[] { + if (runnerErrorCode !== BACK_SWIPE_NO_CHANGE_RUNNER_CODE) return []; + const entries = error?.dismissControls; + if (!Array.isArray(entries)) return []; + return entries.flatMap((entry: unknown): BackDismissControl[] => { + if (typeof entry !== 'object' || entry === null) return []; + const { label, identifier } = entry as { label?: unknown; identifier?: unknown }; + if (typeof label !== 'string') return []; + const id = typeof identifier === 'string' && identifier.length > 0 ? identifier : undefined; + if (id === undefined && label.length === 0) return []; + const selector = + id === undefined ? `role=button label=${JSON.stringify(label)}` : `id=${JSON.stringify(id)}`; + return [{ selector, label, ...(id === undefined ? {} : { identifier: id }) }]; + }); +} + +/** The refusal hint once dismiss controls are known; the runner's own hint otherwise. */ +function backDismissHint(controls: readonly BackDismissControl[]): string | undefined { + const [only] = controls; + if (only === undefined) return undefined; + return controls.length === 1 + ? `Press '${only.selector}' to dismiss this screen; back only pops the navigation stack.` + : 'Press one of details.dismissControls by its selector to dismiss this screen; back only pops the navigation stack.'; +} + /** * The one decoding of a runner response body (#2662). The envelope arrives at three readers — a * command's own response, the lifecycle journal a status probe reads back after the transport @@ -606,15 +647,17 @@ export function buildRunnerResponseError( typeof payload.error?.message === 'string' ? payload.error.message : undefined; const hint = typeof payload.error?.hint === 'string' ? payload.error.hint : undefined; const classification = classifyRunnerReportedError(runnerErrorCode); + const dismissControls = readBackDismissControls(runnerErrorCode, payload.error); return new AppError(classification.code, errorMessage ?? 'Runner error', { runner: payload, ...classification.details, + ...(dismissControls.length > 0 ? { dismissControls } : {}), xcodebuild: { exitCode: 1, stdout: '', stderr: '', }, - hint, + hint: backDismissHint(dismissControls) ?? hint, logPath, }); } diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 8da32ae664..02d08ed66a 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -304,7 +304,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; a visible back or close control is an alternative. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: there back only pops the navigation stack, by its back button or the edge swipe, and never dismisses a modal or sheet, which the app's own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the hittable buttons in the top band of the app window labeled or identified Back, Close, Cancel, or an xmark icon name (never Done or Save), each as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector, such as press 'role=button label=\"Close\"'; back cannot give refs, because it expires the snapshot's refs before it swipes. The field is absent when there is none. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 56f200bae9..e05285cdff 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -126,8 +126,9 @@ iOS Simulator launch behavior: Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. macOS finds that control by the app's own back identifier or "Back" label; an app that exposes neither refuses with `error.details.reason: in_app_back_control_unavailable` under `COMMAND_FAILED` and `dispatched: no`, pressing nothing, so press the app's visible back control by its ref. +- iOS has no system back. On iOS, `back` only pops the navigation stack, by the back button or the edge swipe below. It does not dismiss a modal or a sheet: press that screen's own Close or Cancel control. Android's system back is different: it also closes dialogs and modal screens. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar, or, when none is, one floating below it, such as on the iPhone Duo), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the back buttons and navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. A back button with no title of its own, such as the Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap; when no bar has a title, or a title keeps changing, that button is not pressed and `back` swipes. If none changes, it reports `back_swipe_unverified` and does not swipe. For a back button that has its own title, a bar's title never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself, or a navigation bar that retitles itself under a back button with no title, can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). -- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. Press the visible back or close control. +- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `id="…"` when the control has an identifier, and `role=button label="…"` otherwise. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. When there is none, or the app's tree cannot be read, the field is absent. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons (or, for a back button with no title, the bar's title) did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - Runner log `found=unprovable`: the back button by a navigation bar has no title and no navigation bar has one, so a pop could not be confirmed. The button is not pressed and `back` swipes instead. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From 71705658575a179402607c0342332fe78c52bc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 12:08:12 +0100 Subject: [PATCH 2/5] fix(ios): list only dismiss controls a selector presses alone Read the dismiss controls with an exact predicate and their snapshot frames, ask hittability last, and have the runner name the attribute each control is pressed by. A control whose label or identifier another button shares is left out, since its selector would match both. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Models.swift | 9 ++- .../RunnerTests+Navigation.swift | 68 ++++++++++++------- .../RunnerTests+NavigationTests.swift | 38 +++++++---- apple/runner/RUNNER_PROTOCOL.md | 9 ++- .../runner-back-dismiss-controls.test.ts | 30 +++++--- packages/apple-runner/src/runner-contract.ts | 21 ++++-- packages/command-surface/src/system/index.ts | 2 +- website/docs/docs/commands.md | 2 +- 8 files changed, 118 insertions(+), 61 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index eb29c9c972..15d4a12c33 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -689,10 +689,17 @@ enum RetryableResponseFailure: Equatable { } /// A control that dismisses the screen, such as a modal's Close button. The identifier is absent when -/// the control has none. +/// the control has none. `pressBy` names the attribute no other button shares, which the host turns +/// into the selector that presses it. struct DismissControlPayload: Codable, Equatable { + enum PressBy: String, Codable { + case label + case identifier + } + let label: String let identifier: String? + let pressBy: PressBy } struct ErrorPayload: Codable { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 1446e89f07..3fbf1a1e1c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -676,19 +676,21 @@ extension RunnerTests { return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.") } - /// Narrows the button query to the dismiss names; `isDismissControlName` decides. + /// Matches the buttons `isDismissControlName` accepts, so every button that shares a dismiss + /// control's label or identifier is read too. static let dismissControlPredicate: NSPredicate = { let names = dismissControlWords.sorted() + ["xmark"] - let clauses = names.flatMap { _ in ["label BEGINSWITH[c] %@", "identifier BEGINSWITH[c] %@"] } - return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] }) + let clauses = names.flatMap { _ in ["label ==[c] %@", "identifier ==[c] %@"] } + + ["label BEGINSWITH[c] %@", "identifier BEGINSWITH[c] %@"] + return NSPredicate( + format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] } + ["xmark.", "xmark."]) }() - /// One button the dismiss control read found, with its live frame in screen points. + /// One button the dismiss control read found, with its snapshot frame in the window's own space. struct TopBandButton { let label: String let identifier: String let frame: CGRect - let isHittable: Bool } /// The app window's top band, where a bar or header holds its controls and content does not: @@ -701,22 +703,42 @@ extension RunnerTests { return candidate.midY >= window.minY && candidate.midY <= maxY } - /// The dismiss controls a caller may press: the hittable buttons in the top band of `window` whose - /// label or identifier is a dismiss name, in tree order. The system back button is never one, since - /// back itself presses it. - static func dismissControlCandidates(_ buttons: [TopBandButton], window: CGRect) -> [Int] { - buttons.indices.filter { index in + /// The dismiss controls a caller may press, in tree order: the buttons in the top band of `window` + /// whose label, or else identifier, is a dismiss name, pressed by that attribute. A button that + /// shares that attribute with another button is left out, because no selector presses it alone; so + /// is a system back button, which back's own path presses or declines. `isHittable` is asked last, + /// since each answer is an XCTest query. + static func dismissControls( + _ buttons: [TopBandButton], + window: CGRect, + isHittable: (Int) -> Bool + ) -> [DismissControlPayload] { + func attribute(_ index: Int, _ pressBy: DismissControlPayload.PressBy) -> String { + (pressBy == .label ? buttons[index].label : buttons[index].identifier).lowercased() + } + return buttons.indices.compactMap { index in let button = buttons[index] - return button.isHittable && button.identifier != navigationBarBackButtonIdentifier - && isTopNavigationControlFrame(button.frame, in: window) - && (isDismissControlName(button.label) || isDismissControlName(button.identifier)) + guard button.identifier != navigationBarBackButtonIdentifier, + isTopNavigationControlFrame(button.frame, in: window) else { return nil } + let pressBy: DismissControlPayload.PressBy + if isDismissControlName(button.label) { + pressBy = .label + } else if isDismissControlName(button.identifier) { + pressBy = .identifier + } else { + return nil + } + let shared = buttons.indices.contains { $0 != index && attribute($0, pressBy) == attribute(index, pressBy) } + guard !shared, isHittable(index) else { return nil } + return DismissControlPayload( + label: button.label, identifier: button.identifier.isEmpty ? nil : button.identifier, pressBy: pressBy) } } /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe. It runs /// only on that refusal, so a back that navigates pays nothing for it. Nil when there are none, or - /// when the tree is not readable, a read fails, or a query hangs: the list is advice, and back - /// refuses the same either way. + /// when the tree is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list + /// is advice, and back refuses the same either way. @MainActor private func topBandDismissControls( app: XCUIApplication, @@ -724,7 +746,9 @@ extension RunnerTests { ) -> [DismissControlPayload]? { guard navigationBarBackButtonIsReadable(context: context) else { return nil } let deadlineMs = currentUptimeMs() + Self.navigationBarBackReadBudgetMs - let read = heldNavigationBarRead(.dismissControlLookup) { () -> [TopBandButton]? in + // Snapshot frames are in the window's own space, which a placed iPad window starts off the screen origin. + let window = CGRect(origin: .zero, size: context.geometry.windowOnScreen.rect.size) + let read = heldNavigationBarRead(.dismissControlLookup) { () -> [DismissControlPayload]? in guard let roots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { return nil } let elements = roots.flatMap { $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex @@ -732,16 +756,12 @@ extension RunnerTests { var buttons: [TopBandButton] = [] for element in elements { guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } - buttons.append(TopBandButton( - label: snapshot.label, identifier: snapshot.identifier, frame: element.frame, isHittable: element.isHittable)) + buttons.append(TopBandButton(label: snapshot.label, identifier: snapshot.identifier, frame: snapshot.frame)) } - return buttons - } - guard case .read(let buttons) = read else { return nil } - let controls = Self.dismissControlCandidates(buttons, window: context.geometry.windowOnScreen.rect).map { - DismissControlPayload(label: buttons[$0].label, identifier: buttons[$0].identifier.isEmpty ? nil : buttons[$0].identifier) + return Self.dismissControls(buttons, window: window) { elements[$0].isHittable } } - return controls.isEmpty ? nil : controls + guard case .read(let controls) = read, !controls.isEmpty else { return nil } + return controls } @MainActor diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index c7a524d74d..e9eeee9a48 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -363,7 +363,7 @@ extension RunnerTests { return XCTFail("a bar button titled Back must not count as the back button") } XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange) - XCTAssertEqual(payload.dismissControls, [DismissControlPayload(label: "Back", identifier: nil)]) + XCTAssertEqual(payload.dismissControls, [DismissControlPayload(label: "Back", identifier: nil, pressBy: .label)]) XCTAssertEqual(status.label, "Navigation root") } @@ -906,25 +906,33 @@ extension RunnerTests { } /// The dismiss controls a no-change refusal names: hittable buttons in the window's top band whose - /// label or identifier is a dismiss name. An iPhone 17 Pro window: the band ends at 180 pt. - func testDismissControlCandidatesListOnlyTopBandDismissButtons() { + /// label or identifier is a dismiss name and no other button shares. An iPhone 17 Pro window: the band + /// ends at 180 pt. + func testDismissControlsListOnlyTopBandDismissButtons() { let window = CGRect(x: 0, y: 0, width: 402, height: 874) - func button(_ label: String, identifier: String = "", y: CGFloat = 72, hittable: Bool = true) -> TopBandButton { - TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24), isHittable: hittable) + func button(_ label: String, identifier: String = "", y: CGFloat = 72) -> TopBandButton { + TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24)) + } + func labels(_ buttons: [TopBandButton], hittable: Set? = nil) -> [String] { + Self.dismissControls(buttons, window: window) { hittable?.contains($0) ?? true }.map(\.label) } - XCTAssertEqual(Self.dismissControlCandidates([button("Close")], window: window), [0], "a lone Close in the band") XCTAssertEqual( - Self.dismissControlCandidates([button("Cancel"), button("Title"), button("close")], window: window), [0, 2], - "both candidates are listed, in tree order") - XCTAssertEqual(Self.dismissControlCandidates([button("Done"), button("Save")], window: window), [], "commit words never count") - XCTAssertEqual(Self.dismissControlCandidates([button("Close", y: 400)], window: window), [], "a Close below the band") - XCTAssertEqual(Self.dismissControlCandidates([button("Close", hittable: false)], window: window), []) + Self.dismissControls([button("Close")], window: window) { _ in true }, + [DismissControlPayload(label: "Close", identifier: nil, pressBy: .label)], "a lone Close in the band") + XCTAssertEqual(labels([button("Cancel"), button("Title"), button("close")]), ["Cancel", "close"], "both, in tree order") + XCTAssertEqual(labels([button("Done"), button("Save")]), [], "commit words never count") + XCTAssertEqual(labels([button("Close", y: 400)]), [], "a Close below the band") + XCTAssertEqual(labels([button("Close")], hittable: []), [], "an unhittable Close") + XCTAssertEqual(labels([button("Close account")]), [], "a longer label") XCTAssertEqual( - Self.dismissControlCandidates([button("", identifier: "xmark.circle.fill"), button("Close account")], window: window), [0], - "an xmark icon name counts, a longer label does not") - // The system back button is back's own target, pressed by its own path, so it is never listed. + labels([button("Cancel"), button("cancel", y: 600)]), [], + "a label another button shares is pressed by no selector alone") XCTAssertEqual( - Self.dismissControlCandidates([button("Back", identifier: Self.navigationBarBackButtonIdentifier)], window: window), []) + Self.dismissControls([button("", identifier: "xmark.circle.fill")], window: window) { _ in true }, + [DismissControlPayload(label: "", identifier: "xmark.circle.fill", pressBy: .identifier)], + "an xmark icon identifier is pressed by it") + // The system back button is back's own target, pressed or declined by its own path, so it is never listed. + XCTAssertEqual(labels([button("Back", identifier: Self.navigationBarBackButtonIdentifier)]), []) } #endif diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 65b4be08d8..2f9df9e55f 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -212,17 +212,20 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure | `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe | | `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap | -A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…" }`. +A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…", "pressBy": "label" }`. The `identifier` key is absent when the control has none. +`pressBy` is `label` when the label is the dismiss name, and `identifier` when only the identifier is. The runner fills it after the swipe, from one read of the buttons in the app's windows. That read runs only on this refusal and only when the tree is readable. Its snapshots start for up to 1 s. -An entry is a hittable button whose live frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt. +An entry is a hittable button whose snapshot frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt. Its label or identifier must be `back`, `close`, `cancel`, `xmark`, or an `xmark.` icon name, in any letter case. A `BackButton` is never listed. +A button whose `pressBy` attribute another read button shares, compared without case, is not listed, because a selector would match both. +The runner reads hittability only for the buttons that pass the other checks. The runner never presses these controls. With no entry, or when the read fails or hangs, the key is absent. -The daemon publishes the list as `details.dismissControls` and adds a selector to each entry. +The daemon publishes the list as `details.dismissControls` and adds a selector to each entry: `role=button label="…"` or `id="…"`. The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact diff --git a/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts b/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts index acf4425c4c..3f659d1bd5 100644 --- a/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts +++ b/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts @@ -12,7 +12,7 @@ function backRefusal(code: string, dismissControls?: unknown) { } test('a back swipe that changed nothing publishes its lone dismiss control and names it in the hint', () => { - const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Close' }]); + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Close', pressBy: 'label' }]); assert.equal(error.details?.reason, 'back_swipe_no_change'); assert.deepEqual(error.details?.dismissControls, [ { selector: 'role=button label="Close"', label: 'Close' }, @@ -23,9 +23,9 @@ test('a back swipe that changed nothing publishes its lone dismiss control and n ); }); -test('a dismiss control with an identifier is selected by it', () => { +test('a dismiss control pressed by its identifier is selected by it', () => { const error = backRefusal('BACK_SWIPE_NO_CHANGE', [ - { label: 'xmark', identifier: 'sheet-close' }, + { label: 'xmark', identifier: 'sheet-close', pressBy: 'identifier' }, ]); assert.deepEqual(error.details?.dismissControls, [ { selector: 'id="sheet-close"', label: 'xmark', identifier: 'sheet-close' }, @@ -33,16 +33,26 @@ test('a dismiss control with an identifier is selected by it', () => { }); test('two dismiss controls are both published and the hint points at the list', () => { - const error = backRefusal('BACK_SWIPE_NO_CHANGE', [{ label: 'Cancel' }, { label: 'Close' }]); - assert.deepEqual( - (error.details?.dismissControls as { label: string }[]).map((control) => control.label), - ['Cancel', 'Close'], - ); + const error = backRefusal('BACK_SWIPE_NO_CHANGE', [ + { label: 'Cancel', pressBy: 'label' }, + { label: 'Close', identifier: 'modal-close', pressBy: 'label' }, + ]); + assert.deepEqual(error.details?.dismissControls, [ + { selector: 'role=button label="Cancel"', label: 'Cancel' }, + { selector: 'role=button label="Close"', label: 'Close', identifier: 'modal-close' }, + ]); assert.match(String(error.details?.hint), /one of details\.dismissControls/); }); test('without dismiss controls the refusal keeps the runner hint and publishes no list', () => { - for (const controls of [undefined, [], [{ identifier: 7 }], 'Close']) { + for (const controls of [ + undefined, + [], + [{ identifier: 7 }], + 'Close', + [{ label: 'Close' }], + [{ label: 'Close', pressBy: 'identifier' }], + ]) { const error = backRefusal('BACK_SWIPE_NO_CHANGE', controls); assert.equal(error.details?.dismissControls, undefined); assert.equal(error.details?.hint, RUNNER_HINT); @@ -50,7 +60,7 @@ test('without dismiss controls the refusal keeps the runner hint and publishes n }); test('only the no-change refusal publishes dismiss controls', () => { - const error = backRefusal('BACK_SWIPE_UNVERIFIED', [{ label: 'Close' }]); + const error = backRefusal('BACK_SWIPE_UNVERIFIED', [{ label: 'Close', pressBy: 'label' }]); assert.equal(error.details?.dismissControls, undefined); assert.equal(error.details?.hint, RUNNER_HINT); }); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index c9d86a2391..e067e4bf84 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -568,9 +568,10 @@ export type RunnerResponsePayload = { export type BackDismissControl = Readonly<{ selector: string; label: string; identifier?: string }>; /** - * The dismiss controls the runner names on a `BACK_SWIPE_NO_CHANGE` refusal, each with its selector: - * the identifier when the control has one, otherwise its button role and label. Other codes, and - * entries that are not `{ label, identifier? }` objects, publish nothing. + * The dismiss controls the runner names on a `BACK_SWIPE_NO_CHANGE` refusal, each with its selector. + * The runner's `pressBy` names the attribute no other button shares: `identifier` becomes `id=`, and + * `label` becomes the button role and label. Other codes, and entries that are not + * `{ label, identifier?, pressBy }` objects, publish nothing. */ function readBackDismissControls( runnerErrorCode: string | undefined, @@ -581,12 +582,20 @@ function readBackDismissControls( if (!Array.isArray(entries)) return []; return entries.flatMap((entry: unknown): BackDismissControl[] => { if (typeof entry !== 'object' || entry === null) return []; - const { label, identifier } = entry as { label?: unknown; identifier?: unknown }; + const { label, identifier, pressBy } = entry as { + label?: unknown; + identifier?: unknown; + pressBy?: unknown; + }; if (typeof label !== 'string') return []; const id = typeof identifier === 'string' && identifier.length > 0 ? identifier : undefined; - if (id === undefined && label.length === 0) return []; const selector = - id === undefined ? `role=button label=${JSON.stringify(label)}` : `id=${JSON.stringify(id)}`; + pressBy === 'identifier' && id !== undefined + ? `id=${JSON.stringify(id)}` + : pressBy === 'label' && label.length > 0 + ? `role=button label=${JSON.stringify(label)}` + : undefined; + if (selector === undefined) return []; return [{ selector, label, ...(id === undefined ? {} : { identifier: id }) }]; }); } diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 02d08ed66a..85e1c4a728 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -304,7 +304,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: there back only pops the navigation stack, by its back button or the edge swipe, and never dismisses a modal or sheet, which the app's own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the hittable buttons in the top band of the app window labeled or identified Back, Close, Cancel, or an xmark icon name (never Done or Save), each as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector, such as press 'role=button label=\"Close\"'; back cannot give refs, because it expires the snapshot's refs before it swipes. The field is absent when there is none. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: there back only pops the navigation stack, by its back button or the edge swipe, and never dismisses a modal or sheet, which the app's own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the hittable buttons in the top band of the app window labeled or identified Back, Close, Cancel, or an xmark icon name (never Done or Save), each as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector, such as press 'role=button label=\"Close\"'; back cannot give refs, because it expires the snapshot's refs before it swipes. A control whose label or identifier another button shares is not listed. The field is absent when there is none, or when the tree cannot be read or the read fails. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index e05285cdff..8ffc74ccce 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -128,7 +128,7 @@ Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. macOS finds that control by the app's own back identifier or "Back" label; an app that exposes neither refuses with `error.details.reason: in_app_back_control_unavailable` under `COMMAND_FAILED` and `dispatched: no`, pressing nothing, so press the app's visible back control by its ref. - iOS has no system back. On iOS, `back` only pops the navigation stack, by the back button or the edge swipe below. It does not dismiss a modal or a sheet: press that screen's own Close or Cancel control. Android's system back is different: it also closes dialogs and modal screens. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar, or, when none is, one floating below it, such as on the iPhone Duo), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the back buttons and navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. A back button with no title of its own, such as the Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap; when no bar has a title, or a title keeps changing, that button is not pressed and `back` swipes. If none changes, it reports `back_swipe_unverified` and does not swipe. For a back button that has its own title, a bar's title never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself, or a navigation bar that retitles itself under a back button with no title, can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). -- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `id="…"` when the control has an identifier, and `role=button label="…"` otherwise. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. When there is none, or the app's tree cannot be read, the field is absent. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. +- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `role=button label="…"` when the label is the dismiss word, and `id="…"` when only the identifier is. A control whose label or identifier another button shares is not listed, because its selector would match both. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. The field is absent when there is no control, when the app's tree cannot be read, or when the read fails, hangs, or runs past 1 s. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons (or, for a back button with no title, the bar's title) did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - Runner log `found=unprovable`: the back button by a navigation bar has no title and no navigation bar has one, so a pop could not be confirmed. The button is not pressed and `back` swipes instead. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From 2df99ee8078a80159866196454488e05973123e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 12:20:08 +0100 Subject: [PATCH 3/5] fix(ios): cover dismissControls in the error payload contract and fit back help The every-field ErrorPayload fixture sets dismissControls so the encoder contract covers it. The back description is rewritten shorter than before, so the always-loaded MCP tools stay within their byte budget. Co-Authored-By: Claude Opus 5.5 --- .../UnitTests/RunnerTests+ModelsTests.swift | 3 ++- packages/command-surface/src/system/index.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift index 3ffa22f4ab..981563d98c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+ModelsTests.swift @@ -8,7 +8,8 @@ extension RunnerTests { code: "CODE", message: "message", hint: "hint", - retryableFailure: .targetAppUnavailable + retryableFailure: .targetAppUnavailable, + dismissControls: [DismissControlPayload(label: "Close", identifier: "close", pressBy: .label)] ) let encoded = try JSONEncoder().encode(payload) let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any]) diff --git a/packages/command-surface/src/system/index.ts b/packages/command-surface/src/system/index.ts index 85e1c4a728..2bf08dc1bd 100644 --- a/packages/command-surface/src/system/index.ts +++ b/packages/command-surface/src/system/index.ts @@ -304,7 +304,7 @@ export const SYSTEM_COMMAND_OUTPUT_SCHEMAS = { const appStateCommandDescription = 'Show foreground app/activity (Android and HarmonyOS; iOS and macOS answer per session)'; const backCommandDescription = - "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: there back only pops the navigation stack, by its back button or the edge swipe, and never dismisses a modal or sheet, which the app's own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS both modes press the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeed only when the number of navigation bars, the number of back buttons, or a back button title then changes (for a back button with a title, a bar title or a moved back button alone never counts, so a pop that keeps the same back button title and bar count reports back_swipe_unverified; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap, and a bar that retitles itself can still be misread as a pop; when no bar has a title or a title keeps changing, that button is not pressed); with no such button, two candidates, or an unreadable tree, they swipe from the left edge instead (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeed only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the hittable buttons in the top band of the app window labeled or identified Back, Close, Cancel, or an xmark icon name (never Done or Save), each as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector, such as press 'role=button label=\"Close\"'; back cannot give refs, because it expires the snapshot's refs before it swipes. A control whose label or identifier another button shares is not listed. The field is absent when there is none, or when the tree cannot be read or the read fails. Reason back_swipe_unverified: the button was pressed but the navigation bar's back buttons did not change within 1 s or on one more read after it, or the swipe may or may not have navigated, because the screen kept changing before it or could not be sampled after it; the resulting screen is unconfirmed. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead. A visible back or close control can be targeted directly by its ref."; + "Navigate back in the app or through system navigation. The in-app mode targets the app navigation stack; system targets platform back behavior. iOS has no system back: both modes only pop the navigation stack and never close a modal or sheet, whose own Close or Cancel control does; Android's system back also closes dialogs and modal screens. On iOS, back presses the navigation bar system back button (identifier BackButton) only when exactly one hittable one is found in a readable accessibility tree, and succeeds only when the bar count, the back button count, or a back button title then changes; a back button with no title, such as the iPhone Duo's floating one, is confirmed by its bar's title instead, which must hold still before the tap, and a bar that retitles itself can be misread as a pop. Otherwise it swipes from the left edge (on an iPad whose app window cannot be resolved, only while the app fills the screen) and succeeds only when the app screen changes. Reason back_swipe_no_change: the swipe was sent and the screen did not change; details.dismissControls then lists the top-band Back, Close, Cancel, or xmark buttons (never Done or Save) as { selector, label, identifier? }, and the hint names a lone one. Press one by its selector; back returns no refs, because it expires them. Reason back_swipe_unverified: the button was pressed but the bar did not change within 1 s or on one more read, or the swipe may or may not have navigated because the screen kept changing before it or could not be sampled after it. Reason back_swipe_not_dispatched, or a screen that cannot be sampled before the swipe, means nothing was sent. An app with a left-edge drawer may open the drawer instead."; const homeCommandDescription = 'Send the selected device to its home screen. This leaves the app session open but moves the foreground away from the app.'; const orientationCommandDescription = From c13ce51dfa630bc131a0a4f31c9d20fdd8b54efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 12:51:02 +0100 Subject: [PATCH 4/5] fix(ios): list dismiss controls from the back window with role-scoped selectors Both dismiss control selectors now carry role=button, so press resolves them against buttons only, the set the runner checks for uniqueness; a selectors test pins that scope for the label and id forms. The runner lists controls only from the window back swiped on and checks each label or identifier against the buttons of every window, compared the way the selector engine normalizes text. The test file is renamed for the module it covers. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 86 ++++++++++++------- .../RunnerTests+NavigationTests.swift | 35 ++++++-- apple/runner/RUNNER_PROTOCOL.md | 6 +- ... runner-contract-dismiss-controls.test.ts} | 2 +- packages/apple-runner/src/runner-contract.ts | 7 +- packages/selectors/src/internal/match.test.ts | 36 ++++++++ website/docs/docs/commands.md | 2 +- 7 files changed, 130 insertions(+), 44 deletions(-) rename packages/apple-runner/src/__tests__/{runner-back-dismiss-controls.test.ts => runner-contract-dismiss-controls.test.ts} (96%) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 3fbf1a1e1c..6001421d9c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -666,33 +666,44 @@ extension RunnerTests { return state } - /// A dismiss control's label or identifier, compared without case. These words only leave a screen; - /// "Done", "Save", and the like may commit, so they never count. + /// A dismiss control's label or identifier, compared as `selectorText`. These words only leave a + /// screen; "Done", "Save", and the like may commit, so they never count. static let dismissControlWords: Set = ["back", "close", "cancel"] + /// Text as the selector engine compares it (`normalizeText` in packages/selectors): trimmed, + /// lowercased, with each run of whitespace one space. A listed control's selector matches every + /// button whose label or identifier reads the same this way. + static func selectorText(_ text: String) -> String { + text.split(whereSeparator: \.isWhitespace).joined(separator: " ").lowercased() + } + /// Whether `name` is a dismiss word, or an SF Symbols xmark name such as `xmark.circle.fill`. static func isDismissControlName(_ name: String) -> Bool { - let word = name.lowercased() + let word = selectorText(name) return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.") } - /// Matches the buttons `isDismissControlName` accepts, so every button that shares a dismiss - /// control's label or identifier is read too. + /// Every button whose label or identifier contains a dismiss word, a superset of the buttons a + /// listed control's selector matches: `selectorText` then decides. static let dismissControlPredicate: NSPredicate = { let names = dismissControlWords.sorted() + ["xmark"] - let clauses = names.flatMap { _ in ["label ==[c] %@", "identifier ==[c] %@"] } - + ["label BEGINSWITH[c] %@", "identifier BEGINSWITH[c] %@"] - return NSPredicate( - format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] } + ["xmark.", "xmark."]) + let clauses = names.flatMap { _ in ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"] } + return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] }) }() - /// One button the dismiss control read found, with its snapshot frame in the window's own space. + /// One button the dismiss control read found, with its snapshot frame in its window's own space. struct TopBandButton { let label: String let identifier: String let frame: CGRect } + /// The button's label and identifier, the attributes a dismiss control's selector compares. + struct ButtonName { + let label: String + let identifier: String + } + /// The app window's top band, where a bar or header holds its controls and content does not: /// 22% of the window's height, held between 96 and 180 pt. static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool { @@ -703,20 +714,26 @@ extension RunnerTests { return candidate.midY >= window.minY && candidate.midY <= maxY } - /// The dismiss controls a caller may press, in tree order: the buttons in the top band of `window` - /// whose label, or else identifier, is a dismiss name, pressed by that attribute. A button that - /// shares that attribute with another button is left out, because no selector presses it alone; so - /// is a system back button, which back's own path presses or declines. `isHittable` is asked last, - /// since each answer is an XCTest query. + /// The roots a dismiss control is listed from: the window back swiped on when the context resolved + /// one, else every search root. A control in another window, such as an iPad's second scene, is + /// not on the screen back acted on. + static func dismissControlRoots(resolvedWindow: Root?, searchRoots: [Root]) -> [Root] { + resolvedWindow.map { [$0] } ?? searchRoots + } + + /// The dismiss controls a caller may press, in tree order: the buttons of the back window in the top + /// band of `window` whose label, or else identifier, is a dismiss name, pressed by that attribute. + /// `everyButton` holds the buttons of every window, which the selector `press` resolves can match: + /// a control whose attribute another of them shares is left out, because no selector presses it + /// alone. A system back button is never listed: back's own path presses or declines it. + /// `isHittable` is asked last, since each answer is an XCTest query. static func dismissControls( _ buttons: [TopBandButton], + everyButton: [ButtonName], window: CGRect, isHittable: (Int) -> Bool ) -> [DismissControlPayload] { - func attribute(_ index: Int, _ pressBy: DismissControlPayload.PressBy) -> String { - (pressBy == .label ? buttons[index].label : buttons[index].identifier).lowercased() - } - return buttons.indices.compactMap { index in + buttons.indices.compactMap { index in let button = buttons[index] guard button.identifier != navigationBarBackButtonIdentifier, isTopNavigationControlFrame(button.frame, in: window) else { return nil } @@ -728,17 +745,20 @@ extension RunnerTests { } else { return nil } - let shared = buttons.indices.contains { $0 != index && attribute($0, pressBy) == attribute(index, pressBy) } - guard !shared, isHittable(index) else { return nil } + func attribute(_ name: ButtonName) -> String { selectorText(pressBy == .label ? name.label : name.identifier) } + let text = attribute(ButtonName(label: button.label, identifier: button.identifier)) + let matches = everyButton.filter { attribute($0) == text } + guard matches.count == 1, isHittable(index) else { return nil } return DismissControlPayload( label: button.label, identifier: button.identifier.isEmpty ? nil : button.identifier, pressBy: pressBy) } } - /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe. It runs - /// only on that refusal, so a back that navigates pays nothing for it. Nil when there are none, or - /// when the tree is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list - /// is advice, and back refuses the same either way. + /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe: the back + /// window's buttons, and every root's buttons for the uniqueness check. It runs only on that + /// refusal, so a back that navigates pays nothing for it. Nil when there are none, or when the tree + /// is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list is advice, + /// and back refuses the same either way. @MainActor private func topBandDismissControls( app: XCUIApplication, @@ -749,16 +769,24 @@ extension RunnerTests { // Snapshot frames are in the window's own space, which a placed iPad window starts off the screen origin. let window = CGRect(origin: .zero, size: context.geometry.windowOnScreen.rect.size) let read = heldNavigationBarRead(.dismissControlLookup) { () -> [DismissControlPayload]? in - guard let roots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { return nil } - let elements = roots.flatMap { - $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex + guard let searchRoots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { + return nil + } + @MainActor func matching(_ roots: [XCUIElement]) -> [XCUIElement] { + roots.flatMap { $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex } + } + var everyButton: [ButtonName] = [] + for element in matching(searchRoots) { + guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } + everyButton.append(ButtonName(label: snapshot.label, identifier: snapshot.identifier)) } + let elements = matching(Self.dismissControlRoots(resolvedWindow: context.resolvedWindow, searchRoots: searchRoots)) var buttons: [TopBandButton] = [] for element in elements { guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } buttons.append(TopBandButton(label: snapshot.label, identifier: snapshot.identifier, frame: snapshot.frame)) } - return Self.dismissControls(buttons, window: window) { elements[$0].isHittable } + return Self.dismissControls(buttons, everyButton: everyButton, window: window) { elements[$0].isHittable } } guard case .read(let controls) = read, !controls.isEmpty else { return nil } return controls diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index e9eeee9a48..8bd1f69f3f 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -913,27 +913,48 @@ extension RunnerTests { func button(_ label: String, identifier: String = "", y: CGFloat = 72) -> TopBandButton { TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24)) } - func labels(_ buttons: [TopBandButton], hittable: Set? = nil) -> [String] { - Self.dismissControls(buttons, window: window) { hittable?.contains($0) ?? true }.map(\.label) + func controls(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set? = nil) -> [DismissControlPayload] { + let every = buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) } + others + return Self.dismissControls(buttons, everyButton: every, window: window) { hittable?.contains($0) ?? true } + } + func labels(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set? = nil) -> [String] { + controls(buttons, others: others, hittable: hittable).map(\.label) } XCTAssertEqual( - Self.dismissControls([button("Close")], window: window) { _ in true }, - [DismissControlPayload(label: "Close", identifier: nil, pressBy: .label)], "a lone Close in the band") + controls([button("Close")]), [DismissControlPayload(label: "Close", identifier: nil, pressBy: .label)], + "a lone Close in the band") XCTAssertEqual(labels([button("Cancel"), button("Title"), button("close")]), ["Cancel", "close"], "both, in tree order") XCTAssertEqual(labels([button("Done"), button("Save")]), [], "commit words never count") XCTAssertEqual(labels([button("Close", y: 400)]), [], "a Close below the band") XCTAssertEqual(labels([button("Close")], hittable: []), [], "an unhittable Close") XCTAssertEqual(labels([button("Close account")]), [], "a longer label") XCTAssertEqual( - labels([button("Cancel"), button("cancel", y: 600)]), [], - "a label another button shares is pressed by no selector alone") + labels([button("Cancel"), button(" cancel ", y: 600)]), [], + "a label another button reads the same as, to the selector engine, is pressed by no selector alone") XCTAssertEqual( - Self.dismissControls([button("", identifier: "xmark.circle.fill")], window: window) { _ in true }, + controls([button("", identifier: "xmark.circle.fill")]), [DismissControlPayload(label: "", identifier: "xmark.circle.fill", pressBy: .identifier)], "an xmark icon identifier is pressed by it") // The system back button is back's own target, pressed or declined by its own path, so it is never listed. XCTAssertEqual(labels([button("Back", identifier: Self.navigationBarBackButtonIdentifier)]), []) } + + /// Dismiss controls come from the window back swiped on; another window's buttons are not listed, + /// but a press resolves against every window, so one there that shares a label makes it ambiguous. + func testDismissControlsComeFromTheBackWindowAndAreUniqueAcrossEveryWindow() { + XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: "back window", searchRoots: ["app"]), ["back window"]) + XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: nil, searchRoots: ["left", "right"]), ["left", "right"]) + let window = CGRect(x: 0, y: 0, width: 834, height: 1194) + let close = TopBandButton(label: "Close", identifier: "", frame: CGRect(x: 780, y: 40, width: 30, height: 30)) + let backWindowClose = ButtonName(label: "Close", identifier: "") + XCTAssertEqual( + Self.dismissControls([close], everyButton: [backWindowClose], window: window) { _ in true }.map(\.label), ["Close"]) + XCTAssertEqual( + Self.dismissControls( + [close], everyButton: [backWindowClose, ButtonName(label: "close", identifier: "")], window: window + ) { _ in true }, + [], "a Close in a second window shares the selector") + } #endif /// A still screen settles before the budget runs out at any capture time: the settle loop's diff --git a/apple/runner/RUNNER_PROTOCOL.md b/apple/runner/RUNNER_PROTOCOL.md index 2f9df9e55f..08732576d6 100644 --- a/apple/runner/RUNNER_PROTOCOL.md +++ b/apple/runner/RUNNER_PROTOCOL.md @@ -215,17 +215,17 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…", "pressBy": "label" }`. The `identifier` key is absent when the control has none. `pressBy` is `label` when the label is the dismiss name, and `identifier` when only the identifier is. -The runner fills it after the swipe, from one read of the buttons in the app's windows. +The runner fills it after the swipe. It lists buttons of the window the swipe resolved (every search root when none resolved) and checks uniqueness against the buttons of every search root. That read runs only on this refusal and only when the tree is readable. Its snapshots start for up to 1 s. An entry is a hittable button whose snapshot frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt. Its label or identifier must be `back`, `close`, `cancel`, `xmark`, or an `xmark.` icon name, in any letter case. A `BackButton` is never listed. -A button whose `pressBy` attribute another read button shares, compared without case, is not listed, because a selector would match both. +A button whose `pressBy` attribute another button shares, compared as the selector engine does (trimmed, lowercased, whitespace collapsed), is not listed, because a selector would match both. The runner reads hittability only for the buttons that pass the other checks. The runner never presses these controls. With no entry, or when the read fails or hangs, the key is absent. -The daemon publishes the list as `details.dismissControls` and adds a selector to each entry: `role=button label="…"` or `id="…"`. +The daemon publishes the list as `details.dismissControls` and adds a selector to each entry: `role=button label="…"` or `role=button id="…"`. The `role=button` term limits the match to button elements, the set the runner checked. The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a 0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact diff --git a/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts b/packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts similarity index 96% rename from packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts rename to packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts index 3f659d1bd5..121a6388f1 100644 --- a/packages/apple-runner/src/__tests__/runner-back-dismiss-controls.test.ts +++ b/packages/apple-runner/src/__tests__/runner-contract-dismiss-controls.test.ts @@ -28,7 +28,7 @@ test('a dismiss control pressed by its identifier is selected by it', () => { { label: 'xmark', identifier: 'sheet-close', pressBy: 'identifier' }, ]); assert.deepEqual(error.details?.dismissControls, [ - { selector: 'id="sheet-close"', label: 'xmark', identifier: 'sheet-close' }, + { selector: 'role=button id="sheet-close"', label: 'xmark', identifier: 'sheet-close' }, ]); }); diff --git a/packages/apple-runner/src/runner-contract.ts b/packages/apple-runner/src/runner-contract.ts index e067e4bf84..f71f3f785f 100644 --- a/packages/apple-runner/src/runner-contract.ts +++ b/packages/apple-runner/src/runner-contract.ts @@ -569,8 +569,9 @@ export type BackDismissControl = Readonly<{ selector: string; label: string; ide /** * The dismiss controls the runner names on a `BACK_SWIPE_NO_CHANGE` refusal, each with its selector. - * The runner's `pressBy` names the attribute no other button shares: `identifier` becomes `id=`, and - * `label` becomes the button role and label. Other codes, and entries that are not + * The runner's `pressBy` names the attribute no other button shares. Both forms carry `role=button`, + * which scopes the match to the buttons the runner checked, and resolves through the snapshot tree: + * `identifier` becomes `role=button id="…"`, and `label` becomes `role=button label="…"`. Other codes, and entries that are not * `{ label, identifier?, pressBy }` objects, publish nothing. */ function readBackDismissControls( @@ -591,7 +592,7 @@ function readBackDismissControls( const id = typeof identifier === 'string' && identifier.length > 0 ? identifier : undefined; const selector = pressBy === 'identifier' && id !== undefined - ? `id=${JSON.stringify(id)}` + ? `role=button id=${JSON.stringify(id)}` : pressBy === 'label' && label.length > 0 ? `role=button label=${JSON.stringify(label)}` : undefined; diff --git a/packages/selectors/src/internal/match.test.ts b/packages/selectors/src/internal/match.test.ts index 18ce660854..09d8e7233c 100644 --- a/packages/selectors/src/internal/match.test.ts +++ b/packages/selectors/src/internal/match.test.ts @@ -280,3 +280,39 @@ test('appName selector matches nodes with appName field', () => { assert.ok(match3); assert.equal(match3.matches, 1); }); + +// iOS `back` lists a dismiss control with a role-scoped selector after the runner checked that no +// other button shares its label or identifier; the role term is what keeps non-buttons out. +test('role=button scopes a dismiss control selector to buttons, so same-text non-buttons stay out', () => { + const node = (ref: string, type: string, label: string, identifier = '') => ({ + ref, + index: Number(ref.slice(1)), + type, + kind: formatRole(type), + label, + identifier, + rect: { x: 366, y: 72, width: 24, height: 24 }, + enabled: true, + hittable: true, + }); + const nodes: SnapshotState['nodes'] = [ + node('n0', 'XCUIElementTypeStaticText', 'Close', 'modal-close'), + node('n1', 'XCUIElementTypeImage', 'close', 'modal-close'), + node('n2', 'XCUIElementTypeButton', 'Close', 'modal-close'), + ]; + for (const selector of ['role=button label="Close"', 'role=button id="modal-close"']) { + const resolved = resolveSelectorChain(nodes, parseSelectorChain(selector), { + platform: 'ios', + requireUnique: true, + }); + assert.equal(resolved?.node.ref, 'n2', selector); + } + // Without the role term the same label is ambiguous across the three element types. + assert.equal( + resolveSelectorChain(nodes, parseSelectorChain('label="Close"'), { + platform: 'ios', + requireUnique: true, + }), + null, + ); +}); diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 8ffc74ccce..c4a93be7ad 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -128,7 +128,7 @@ Back, orientation, and hardware buttons: - `back` uses app-owned back navigation: the back keyevent on Android, the visible back control on macOS, and the Siri Remote Menu button on tvOS. `--in-app` names this default explicitly. macOS finds that control by the app's own back identifier or "Back" label; an app that exposes neither refuses with `error.details.reason: in_app_back_control_unavailable` under `COMMAND_FAILED` and `dispatched: no`, pressing nothing, so press the app's visible back control by its ref. - iOS has no system back. On iOS, `back` only pops the navigation stack, by the back button or the edge swipe below. It does not dismiss a modal or a sheet: press that screen's own Close or Cancel control. Android's system back is different: it also closes dialogs and modal screens. - On iOS, `back` first looks for the navigation bar's system back button (a button with the identifier `BackButton` inside a navigation bar, or, when none is, one floating below it, such as on the iPhone Duo), read fresh in the same call. It presses that button only when exactly one live, hittable one is found, the app's accessibility tree is readable (it is not read when XCTest could not query it earlier in the session), and two reads of the back buttons and navigation bars at least 0.05 s apart agree, so a screen that is still being pushed is not taken for the screen to go back from. After the tap, `back` reads the navigation bars again, about every 0.05 s, and succeeds once one of these changes: the number of navigation bars, the number of back buttons (for example, the button is gone at the root screen), or a back button's title. A back button with no title of its own, such as the Duo's floating one, is confirmed by its navigation bar's title instead, which must also hold still before the tap; when no bar has a title, or a title keeps changing, that button is not pressed and `back` swipes. If none changes, it reports `back_swipe_unverified` and does not swipe. For a back button that has its own title, a bar's title never counts, because a title can update itself, such as a countdown timer. A back button that only moves never counts either, because a tap that goes nowhere can still move it, such as when a large title collapses. So a pop that leaves the back button as it was, for example between two screens whose back buttons show the same title (such as an app that titles every back button "Back"), also reports `back_swipe_unverified`. A back button whose own title updates itself, or a navigation bar that retitles itself under a back button with no title, can still be misread as a navigation. Before and after the tap, reads start for up to 1 s each; a read that is still running at that point finishes, or ends at XCTest's own query timeout. A read that is still running can take well past the 1 s; about 2 s was measured on a CI simulator. After the tap, when no read in that 1 s saw a change, `back` starts one more read after it, so a pop can still be confirmed on a slow host. That read can also fail, or hang until XCTest's query timeout; both report `back_swipe_unverified`. When an XCTest query hangs, `back` sends nothing more: before the tap it reports `back_swipe_not_dispatched`, and after the tap `back_swipe_unverified`. Only the system back button counts: a custom bar button titled "Back", or a "Back" button in a toolbar, is never pressed. With no such button, with two or more (such as a sheet over a stack), when the tree cannot be read, when the bars do not hold still within 1 s, or when the tap is refused before it is sent, `back` is the left-edge back swipe, with or without `--in-app` or `--system`. When XCTest cannot resolve the app's window, the swipe samples the device's only screen: on a single-screen iPhone, or on an iPad only while the app provably fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. The swipe succeeds only when the app's screen, drawn as a 48×96 grayscale grid with its top 7 rows left out (in portrait, the status bar and Dynamic Island band, whether the status bar is shown or hidden; in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows), differs in more than 1% of the remaining cells by more than 24 of 256 levels. A small spinner usually stays under that; a larger animation can still count as a change. The change counted is the one between the sample taken immediately before the swipe and the one taken about 0.1 s after it, so a screen that changes on its own inside that gap, after it looked still, can be misread as a back navigation; verify with `snapshot` when the screen has auto-advancing content. Before it swipes, it waits for the screen to stop changing, with a 0.05 s pause between captures (each capture adds its own time), within a settle budget of 2 s that extends on a host whose captures are slow enough that a still screen needs more time. The screen must stay still for the contact allowance (0.2 s on a simulator, 0.4 s on a physical device), a 0.25 s margin, and the slowest capture: about 0.55 s in all on a simulator. The margin keeps a screen that repaints itself about every 0.5 s, such as a carousel, from looking still, so its next repaint is not read as a back; it does not depend on the 0.1 s wait after the swipe. A screen that repaints on its own more often than the still span never settles, and `back` reports `back_swipe_unverified`; the runner's tests check this with a screen that repaints every 0.45 s. A still screen usually settles in about 0.6 s on a simulator, and in about 1.4 s on a physical iPhone, based on the 0.4 s capture measured on one iPhone 17 Pro with iOS 27.0.1 ([PR #4460 evidence](https://github.com/callstack/agent-device/pull/4460#issuecomment-6828546541)). -- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `role=button label="…"` when the label is the dismiss word, and `id="…"` when only the identifier is. A control whose label or identifier another button shares is not listed, because its selector would match both. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. The field is absent when there is no control, when the app's tree cannot be read, or when the read fails, hangs, or runs past 1 s. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. +- `reason: back_swipe_no_change`: the swipe was sent and the screen did not change. `details.dismissControls` then lists the controls that can leave the screen: the hittable buttons in the top band of the app window (the top 22% of its height, at least 96 pt and at most 180 pt) whose label or identifier is Back, Close, Cancel, or an `xmark` icon name, in any letter case. Done, Save, and other labels are never listed. Each entry is `{ selector, label, identifier? }`. Press one with its selector, for example `agent-device press 'role=button label="Close"'`; no new snapshot is needed. The selector is `role=button label="…"` when the label is the dismiss word, and `role=button id="…"` when only the identifier is. Only controls in the app window that `back` swiped are listed. A control whose label or identifier another button in any of the app's windows shares (compared without letter case or extra spaces) is not listed, because its selector would match both. `back` cannot return refs, because it expires the snapshot's refs before it swipes. When there is exactly one control, the hint names it. The field is absent when there is no control, when the app's tree cannot be read, or when the read fails, hangs, or runs past 1 s. `back` never presses these controls itself, because Close or Cancel can discard a draft or end a flow. - `reason: back_swipe_unverified`: the back button was pressed but the navigation bar's back buttons (or, for a back button with no title, the bar's title) did not change within 1 s or on one more read after it or an XCTest query hung after the tap, or the swipe may or may not have navigated, because the screen kept changing before the swipe, the swipe took longer than the screen had been still, or the screen could not be sampled after it. Check with `snapshot`. A real pop on a stack of three or more screens also answers this after the full 1 s when the outgoing and incoming back buttons share a label. - Runner log `found=unprovable`: the back button by a navigation bar has no title and no navigation bar has one, so a pop could not be confirmed. The button is not pressed and `back` swipes instead. - `reason: back_swipe_not_dispatched`: the swipe was refused before any touch went down, or an XCTest query hung before the back button tap, so nothing reached the device. From ca13deb8af9b5945ddd87cc81bbb149afe62d104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 13:14:08 +0100 Subject: [PATCH 5/5] fix(ios): judge dismiss control uniqueness on the label press resolves The uniqueness read now compares each button's snapshot label, which falls back to its value or a child's text as press sees it, and reads unlabeled buttons for that reason. A shared normalization table ties the runner's selectorText to the selector engine's normalizeText. With no resolved window the first read also serves as the listing read. Co-Authored-By: Claude Opus 5.5 --- .../RunnerTests+Navigation.swift | 46 +++++++++++-------- .../RunnerTests+NavigationTests.swift | 10 ++++ .../fixtures/selector-text-normalization.json | 19 ++++++++ .../src/internal/find-normalize-text.test.ts | 29 ++++++++++++ 4 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 contracts/fixtures/selector-text-normalization.json create mode 100644 packages/selectors/src/internal/find-normalize-text.test.ts diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift index 6001421d9c..7376f426a1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift @@ -683,15 +683,18 @@ extension RunnerTests { return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.") } - /// Every button whose label or identifier contains a dismiss word, a superset of the buttons a - /// listed control's selector matches: `selectorText` then decides. + /// Every button whose label or identifier contains a dismiss word, and every unlabeled one, whose + /// snapshot label can come from a child's text: a superset of the buttons a listed control's + /// selector matches. `selectorText` then decides on the label `press` resolves. static let dismissControlPredicate: NSPredicate = { let names = dismissControlWords.sorted() + ["xmark"] - let clauses = names.flatMap { _ in ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"] } + let clauses = names.flatMap { _ in ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"] } + ["label == ''"] return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] }) }() - /// One button the dismiss control read found, with its snapshot frame in its window's own space. + /// One button the dismiss control read found: the label and identifier a snapshot node publishes for + /// it (`evaluateSnapshot`), which `press` resolves selectors against, and its snapshot frame in its + /// window's own space. struct TopBandButton { let label: String let identifier: String @@ -754,8 +757,9 @@ extension RunnerTests { } } - /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe: the back - /// window's buttons, and every root's buttons for the uniqueness check. It runs only on that + /// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe: every root's + /// buttons for the uniqueness check, and the back window's buttons, which are the same read when no + /// window resolved. It runs only on that /// refusal, so a back that navigates pays nothing for it. Nil when there are none, or when the tree /// is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list is advice, /// and back refuses the same either way. @@ -772,21 +776,25 @@ extension RunnerTests { guard let searchRoots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else { return nil } - @MainActor func matching(_ roots: [XCUIElement]) -> [XCUIElement] { - roots.flatMap { $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex } - } - var everyButton: [ButtonName] = [] - for element in matching(searchRoots) { - guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } - everyButton.append(ButtonName(label: snapshot.label, identifier: snapshot.identifier)) + @MainActor func buttons(under roots: [XCUIElement]) -> (elements: [XCUIElement], buttons: [TopBandButton])? { + let elements = roots.flatMap { + $0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex + } + var buttons: [TopBandButton] = [] + for element in elements { + guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } + let node = evaluateSnapshot(snapshot) + buttons.append(TopBandButton(label: node.label, identifier: node.identifier, frame: snapshot.frame)) + } + return (elements, buttons) } - let elements = matching(Self.dismissControlRoots(resolvedWindow: context.resolvedWindow, searchRoots: searchRoots)) - var buttons: [TopBandButton] = [] - for element in elements { - guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil } - buttons.append(TopBandButton(label: snapshot.label, identifier: snapshot.identifier, frame: snapshot.frame)) + guard let every = buttons(under: searchRoots) else { return nil } + let roots = Self.dismissControlRoots(resolvedWindow: context.resolvedWindow, searchRoots: searchRoots) + guard let listed = context.resolvedWindow == nil ? every : buttons(under: roots) else { return nil } + let everyButton = every.buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) } + return Self.dismissControls(listed.buttons, everyButton: everyButton, window: window) { + listed.elements[$0].isHittable } - return Self.dismissControls(buttons, everyButton: everyButton, window: window) { elements[$0].isHittable } } guard case .read(let controls) = read, !controls.isEmpty else { return nil } return controls diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift index 8bd1f69f3f..3e0358c2e1 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+NavigationTests.swift @@ -939,6 +939,16 @@ extension RunnerTests { XCTAssertEqual(labels([button("Back", identifier: Self.navigationBarBackButtonIdentifier)]), []) } + /// The runner's text comparison is the selector engine's `normalizeText`: both are asserted on + /// contracts/fixtures/selector-text-normalization.json, so a listed selector is unique to `press`. + func testSelectorTextMatchesTheSelectorEngineNormalization() throws { + let cases = try XCTUnwrap(runnerContractFixture("selector-text-normalization.json") as? [[String: String]]) + XCTAssertFalse(cases.isEmpty) + for fixture in cases { + XCTAssertEqual(Self.selectorText(try XCTUnwrap(fixture["input"])), fixture["normalized"], fixture["name"] ?? "") + } + } + /// Dismiss controls come from the window back swiped on; another window's buttons are not listed, /// but a press resolves against every window, so one there that shares a label makes it ambiguous. func testDismissControlsComeFromTheBackWindowAndAreUniqueAcrossEveryWindow() { diff --git a/contracts/fixtures/selector-text-normalization.json b/contracts/fixtures/selector-text-normalization.json new file mode 100644 index 0000000000..6f40de8d4b --- /dev/null +++ b/contracts/fixtures/selector-text-normalization.json @@ -0,0 +1,19 @@ +[ + { + "name": "surrounding spaces and a tab are trimmed, case is folded", + "input": " Cancel\t", + "normalized": "cancel" + }, + { + "name": "a run of whitespace inside the text is one space", + "input": "Close\n\n Now", + "normalized": "close now" + }, + { "name": "no-break spaces count as whitespace", "input": " Back ", "normalized": "back" }, + { + "name": "an icon name keeps its dots", + "input": "XMARK.circle.fill", + "normalized": "xmark.circle.fill" + }, + { "name": "whitespace only is empty", "input": " \t\n", "normalized": "" } +] diff --git a/packages/selectors/src/internal/find-normalize-text.test.ts b/packages/selectors/src/internal/find-normalize-text.test.ts new file mode 100644 index 0000000000..b6f6cdc8bc --- /dev/null +++ b/packages/selectors/src/internal/find-normalize-text.test.ts @@ -0,0 +1,29 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { normalizeText } from './find.ts'; + +// Golden text normalization table: the Swift runner's `selectorText`, which decides whether iOS back +// lists a dismiss control's selector as unique, is asserted against the same JSON +// (UnitTests/RunnerTests+NavigationTests.swift). +type NormalizationCase = { name: string; input: string; normalized: string }; + +const TABLE_PATH = path.resolve( + import.meta.dirname, + '..', + '..', + '..', + '..', + 'contracts', + 'fixtures', + 'selector-text-normalization.json', +); + +test('selector text normalization matches the shared table', () => { + const cases = JSON.parse(fs.readFileSync(TABLE_PATH, 'utf8')) as NormalizationCase[]; + assert.ok(cases.length > 0, 'the normalization table must not be empty'); + for (const fixture of cases) { + assert.equal(normalizeText(fixture.input), fixture.normalized, fixture.name); + } +});