Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -688,18 +688,36 @@ enum RetryableResponseFailure: Equatable {
case targetAppUnavailable
}

/// A control that dismisses the screen, such as a modal's Close button. The identifier is absent when
/// the control has none. `pressBy` names the attribute no other button shares, which the host turns
/// into the selector that presses it.
struct DismissControlPayload: Codable, Equatable {
enum PressBy: String, Codable {
case label
case identifier
}

let label: String
let identifier: String?
let pressBy: PressBy
}

struct ErrorPayload: Codable {
var code: String?
let message: String
var hint: String?
/// Runner-internal: read by `shouldRetryResponse` and never encoded, so the host's decoding of
/// the error is unchanged.
var retryableFailure: RetryableResponseFailure? = nil
/// Only on an iOS `BACK_SWIPE_NO_CHANGE`: the dismiss controls in the top band the caller may press
/// instead. Absent when there are none.
var dismissControls: [DismissControlPayload]? = nil

private enum CodingKeys: String, CodingKey {
case code
case message
case hint
case dismissControls
}

static func targetAppUnavailable(bundleId: String?) -> ErrorPayload {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,9 @@ extension RunnerTests {

/// iOS `back` presses the navigation bar's system back button when exactly one is on screen, read
/// from the live tree in this call, and confirms that the bar changed. Otherwise it is the
/// left-edge back swipe, verified by a change of the app's screen. No other button counts,
/// whatever its label.
/// left-edge back swipe, verified by a change of the app's screen. No other button is pressed,
/// whatever its label: a swipe that changed nothing names the dismiss controls the caller may press
/// instead (`topBandDismissControls`).
@MainActor
func performInAppBack(app: XCUIApplication) throws -> InAppBackOutcome {
#if os(macOS)
Expand Down Expand Up @@ -137,8 +138,12 @@ extension RunnerTests {
return pressed
}
do {
let outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace)
var outcome = try verifiedBackSwipe(app: app, context: context, trace: &trace)
logSwipe(trace.logLine(outcome: outcome))
if case .unconfirmed(var payload) = outcome, payload.code == RunnerWireErrorCode.backSwipeNoChange {
payload.dismissControls = topBandDismissControls(app: app, context: context)
outcome = .unconfirmed(payload)
}
return outcome
} catch {
logSwipe(trace.logLine(name: "error", code: String(describing: type(of: error))))
Expand Down Expand Up @@ -508,12 +513,14 @@ extension RunnerTests {
return !isSnapshotXCTestChannelPenalized(bundleId: mainOwned.bundleId)
}

/// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, or the
/// final confirmation read. The read passes it to the test read hook.
/// The phase a bar read belongs to: the lookup before the tap, a confirmation read after it, the
/// final confirmation read, or the dismiss control read after a swipe that changed nothing. The
/// read passes it to the test read hook.
enum NavigationBarBackReadPhase {
case lookup
case confirmation
case finalConfirmation
case dismissControlLookup
}

/// One read of the bar: its value, a read that proved nothing, or an XCTest query that hung.
Expand Down Expand Up @@ -659,6 +666,140 @@ extension RunnerTests {
return state
}

/// A dismiss control's label or identifier, compared as `selectorText`. These words only leave a
/// screen; "Done", "Save", and the like may commit, so they never count.
static let dismissControlWords: Set<String> = ["back", "close", "cancel"]

/// Text as the selector engine compares it (`normalizeText` in packages/selectors): trimmed,
/// lowercased, with each run of whitespace one space. A listed control's selector matches every
/// button whose label or identifier reads the same this way.
static func selectorText(_ text: String) -> String {
text.split(whereSeparator: \.isWhitespace).joined(separator: " ").lowercased()
}

/// Whether `name` is a dismiss word, or an SF Symbols xmark name such as `xmark.circle.fill`.
static func isDismissControlName(_ name: String) -> Bool {
let word = selectorText(name)
return dismissControlWords.contains(word) || word == "xmark" || word.hasPrefix("xmark.")
}

/// Every button whose label or identifier contains a dismiss word, and every unlabeled one, whose
/// snapshot label can come from a child's text: a superset of the buttons a listed control's
/// selector matches. `selectorText` then decides on the label `press` resolves.
static let dismissControlPredicate: NSPredicate = {
let names = dismissControlWords.sorted() + ["xmark"]
let clauses = names.flatMap { _ in ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"] } + ["label == ''"]
return NSPredicate(format: clauses.joined(separator: " OR "), argumentArray: names.flatMap { [$0, $0] })
}()

/// One button the dismiss control read found: the label and identifier a snapshot node publishes for
/// it (`evaluateSnapshot`), which `press` resolves selectors against, and its snapshot frame in its
/// window's own space.
struct TopBandButton {
let label: String
let identifier: String
let frame: CGRect
}

/// The button's label and identifier, the attributes a dismiss control's selector compares.
struct ButtonName {
let label: String
let identifier: String
}

/// The app window's top band, where a bar or header holds its controls and content does not:
/// 22% of the window's height, held between 96 and 180 pt.
static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window) else {
return false
}
let maxY = window.minY + min(max(window.height * 0.22, 96), 180)
return candidate.midY >= window.minY && candidate.midY <= maxY
}

/// The roots a dismiss control is listed from: the window back swiped on when the context resolved
/// one, else every search root. A control in another window, such as an iPad's second scene, is
/// not on the screen back acted on.
static func dismissControlRoots<Root>(resolvedWindow: Root?, searchRoots: [Root]) -> [Root] {
resolvedWindow.map { [$0] } ?? searchRoots
}

/// The dismiss controls a caller may press, in tree order: the buttons of the back window in the top
/// band of `window` whose label, or else identifier, is a dismiss name, pressed by that attribute.
/// `everyButton` holds the buttons of every window, which the selector `press` resolves can match:
/// a control whose attribute another of them shares is left out, because no selector presses it
/// alone. A system back button is never listed: back's own path presses or declines it.
/// `isHittable` is asked last, since each answer is an XCTest query.
static func dismissControls(
_ buttons: [TopBandButton],
everyButton: [ButtonName],
window: CGRect,
isHittable: (Int) -> Bool
) -> [DismissControlPayload] {
buttons.indices.compactMap { index in
let button = buttons[index]
guard button.identifier != navigationBarBackButtonIdentifier,
isTopNavigationControlFrame(button.frame, in: window) else { return nil }
let pressBy: DismissControlPayload.PressBy
if isDismissControlName(button.label) {
pressBy = .label
} else if isDismissControlName(button.identifier) {
pressBy = .identifier
} else {
return nil
}
func attribute(_ name: ButtonName) -> String { selectorText(pressBy == .label ? name.label : name.identifier) }
let text = attribute(ButtonName(label: button.label, identifier: button.identifier))
let matches = everyButton.filter { attribute($0) == text }
guard matches.count == 1, isHittable(index) else { return nil }
return DismissControlPayload(
label: button.label, identifier: button.identifier.isEmpty ? nil : button.identifier, pressBy: pressBy)
}
}

/// The dismiss controls for a `BACK_SWIPE_NO_CHANGE` refusal, read once after the swipe: every root's
/// buttons for the uniqueness check, and the back window's buttons, which are the same read when no
/// window resolved. It runs only on that
/// refusal, so a back that navigates pays nothing for it. Nil when there are none, or when the tree
/// is not readable, a read fails, its 1 s budget runs out, or a query hangs: the list is advice,
/// and back refuses the same either way.
@MainActor
private func topBandDismissControls(
app: XCUIApplication,
context: SynthesizedCoordinateContext
) -> [DismissControlPayload]? {
guard navigationBarBackButtonIsReadable(context: context) else { return nil }
let deadlineMs = currentUptimeMs() + Self.navigationBarBackReadBudgetMs
// Snapshot frames are in the window's own space, which a placed iPad window starts off the screen origin.
let window = CGRect(origin: .zero, size: context.geometry.windowOnScreen.rect.size)
let read = heldNavigationBarRead(.dismissControlLookup) { () -> [DismissControlPayload]? in
guard let searchRoots = try? elementSearchRoots(root: app, litDisplayID: requestLitPanel?.displayID) else {
return nil
}
@MainActor func buttons(under roots: [XCUIElement]) -> (elements: [XCUIElement], buttons: [TopBandButton])? {
let elements = roots.flatMap {
$0.descendants(matching: .button).matching(Self.dismissControlPredicate).allElementsBoundByIndex
}
var buttons: [TopBandButton] = []
for element in elements {
guard currentUptimeMs() < deadlineMs, let snapshot = try? element.snapshot() else { return nil }
let node = evaluateSnapshot(snapshot)
buttons.append(TopBandButton(label: node.label, identifier: node.identifier, frame: snapshot.frame))
}
return (elements, buttons)
}
guard let every = buttons(under: searchRoots) else { return nil }
let roots = Self.dismissControlRoots(resolvedWindow: context.resolvedWindow, searchRoots: searchRoots)
guard let listed = context.resolvedWindow == nil ? every : buttons(under: roots) else { return nil }
let everyButton = every.buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) }
return Self.dismissControls(listed.buttons, everyButton: everyButton, window: window) {
listed.elements[$0].isHittable
}
}
guard case .read(let controls) = read, !controls.isEmpty else { return nil }
return controls
}

@MainActor
private func verifiedBackSwipe(
app: XCUIApplication,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ extension RunnerTests {
code: "CODE",
message: "message",
hint: "hint",
retryableFailure: .targetAppUnavailable
retryableFailure: .targetAppUnavailable,
dismissControls: [DismissControlPayload(label: "Close", identifier: "close", pressBy: .label)]
)
let encoded = try JSONEncoder().encode(payload)
let object = try XCTUnwrap(JSONSerialization.jsonObject(with: encoded) as? [String: Any])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,7 @@ extension RunnerTests {
case .finalConfirmation:
finalReads += 1
return nil
case .dismissControlLookup: return nil
}
}
defer { navigationBarBackReadForTesting = nil }
Expand All @@ -352,7 +353,7 @@ extension RunnerTests {
}

/// A bar button titled "Back" is not the system back button, so back never presses it: it swipes,
/// and the swipe that changes nothing reports so.
/// and the swipe that changes nothing reports so and names that button for the caller to press.
@MainActor
func testInAppBackNeverPressesABarButtonOnlyTitledBack() throws {
let status = try launchBackNavigationFixture(["--agent-device-back-navigation-bar", "--custom-back-item"])
Expand All @@ -362,6 +363,7 @@ extension RunnerTests {
return XCTFail("a bar button titled Back must not count as the back button")
}
XCTAssertEqual(payload.code, RunnerWireErrorCode.backSwipeNoChange)
XCTAssertEqual(payload.dismissControls, [DismissControlPayload(label: "Back", identifier: nil, pressBy: .label)])
XCTAssertEqual(status.label, "Navigation root")
}

Expand Down Expand Up @@ -902,6 +904,67 @@ extension RunnerTests {
XCTAssertEqual(Self.backButtonCandidates([below, content], barFrames: [bar], windowHeight: 874), [0])
XCTAssertEqual(Self.backButtonCandidates([content], barFrames: [bar], windowHeight: 874), [])
}

/// The dismiss controls a no-change refusal names: hittable buttons in the window's top band whose
/// label or identifier is a dismiss name and no other button shares. An iPhone 17 Pro window: the band
/// ends at 180 pt.
func testDismissControlsListOnlyTopBandDismissButtons() {
let window = CGRect(x: 0, y: 0, width: 402, height: 874)
func button(_ label: String, identifier: String = "", y: CGFloat = 72) -> TopBandButton {
TopBandButton(label: label, identifier: identifier, frame: CGRect(x: 366, y: y, width: 24, height: 24))
}
func controls(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set<Int>? = nil) -> [DismissControlPayload] {
let every = buttons.map { ButtonName(label: $0.label, identifier: $0.identifier) } + others
return Self.dismissControls(buttons, everyButton: every, window: window) { hittable?.contains($0) ?? true }
}
func labels(_ buttons: [TopBandButton], others: [ButtonName] = [], hittable: Set<Int>? = nil) -> [String] {
controls(buttons, others: others, hittable: hittable).map(\.label)
}
XCTAssertEqual(
controls([button("Close")]), [DismissControlPayload(label: "Close", identifier: nil, pressBy: .label)],
"a lone Close in the band")
XCTAssertEqual(labels([button("Cancel"), button("Title"), button("close")]), ["Cancel", "close"], "both, in tree order")
XCTAssertEqual(labels([button("Done"), button("Save")]), [], "commit words never count")
XCTAssertEqual(labels([button("Close", y: 400)]), [], "a Close below the band")
XCTAssertEqual(labels([button("Close")], hittable: []), [], "an unhittable Close")
XCTAssertEqual(labels([button("Close account")]), [], "a longer label")
XCTAssertEqual(
labels([button("Cancel"), button(" cancel ", y: 600)]), [],
"a label another button reads the same as, to the selector engine, is pressed by no selector alone")
XCTAssertEqual(
controls([button("", identifier: "xmark.circle.fill")]),
[DismissControlPayload(label: "", identifier: "xmark.circle.fill", pressBy: .identifier)],
"an xmark icon identifier is pressed by it")
// The system back button is back's own target, pressed or declined by its own path, so it is never listed.
XCTAssertEqual(labels([button("Back", identifier: Self.navigationBarBackButtonIdentifier)]), [])
}

/// The runner's text comparison is the selector engine's `normalizeText`: both are asserted on
/// contracts/fixtures/selector-text-normalization.json, so a listed selector is unique to `press`.
func testSelectorTextMatchesTheSelectorEngineNormalization() throws {
let cases = try XCTUnwrap(runnerContractFixture("selector-text-normalization.json") as? [[String: String]])
XCTAssertFalse(cases.isEmpty)
for fixture in cases {
XCTAssertEqual(Self.selectorText(try XCTUnwrap(fixture["input"])), fixture["normalized"], fixture["name"] ?? "")
}
}

/// Dismiss controls come from the window back swiped on; another window's buttons are not listed,
/// but a press resolves against every window, so one there that shares a label makes it ambiguous.
func testDismissControlsComeFromTheBackWindowAndAreUniqueAcrossEveryWindow() {
XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: "back window", searchRoots: ["app"]), ["back window"])
XCTAssertEqual(Self.dismissControlRoots(resolvedWindow: nil, searchRoots: ["left", "right"]), ["left", "right"])
let window = CGRect(x: 0, y: 0, width: 834, height: 1194)
let close = TopBandButton(label: "Close", identifier: "", frame: CGRect(x: 780, y: 40, width: 30, height: 30))
let backWindowClose = ButtonName(label: "Close", identifier: "")
XCTAssertEqual(
Self.dismissControls([close], everyButton: [backWindowClose], window: window) { _ in true }.map(\.label), ["Close"])
XCTAssertEqual(
Self.dismissControls(
[close], everyButton: [backWindowClose, ButtonName(label: "close", identifier: "")], window: window
) { _ in true },
[], "a Close in a second window shares the selector")
}
#endif

/// A still screen settles before the budget runs out at any capture time: the settle loop's
Expand Down
15 changes: 15 additions & 0 deletions apple/runner/RUNNER_PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,21 @@ fills its screen; otherwise it refuses with `APP_SCREEN_UNRESOLVED`. Its failure
| `BACK_SWIPE_NO_CHANGE` | The swipe was sent and the sampled grid (48×96 grayscale cells, with the top 7 rows always left out; in portrait they hold the status bar and Dynamic Island, and in landscape the iPhone status bar is hidden and the Dynamic Island sits at a side edge, outside those rows) changed in at most 1% of the remaining cells by more than 24 of 256 levels. | The swipe |
| `BACK_SWIPE_UNVERIFIED` | The swipe may have run, or the back button was tapped and the bar's back buttons did not change within 1 s or on one more read after it or a query on it hung; the outcome is unknown. | Possibly the swipe, or the tap |

A `BACK_SWIPE_NO_CHANGE` error can carry `dismissControls`, an array of `{ "label": "Close", "identifier": "…", "pressBy": "label" }`.
The `identifier` key is absent when the control has none.
`pressBy` is `label` when the label is the dismiss name, and `identifier` when only the identifier is.
The runner fills it after the swipe. It lists buttons of the window the swipe resolved (every search root when none resolved) and checks uniqueness against the buttons of every search root.
That read runs only on this refusal and only when the tree is readable.
Its snapshots start for up to 1 s.
An entry is a hittable button whose snapshot frame has its center in the window's top band, which is 22% of the window height, at least 96 pt and at most 180 pt.
Its label or identifier must be `back`, `close`, `cancel`, `xmark`, or an `xmark.` icon name, in any letter case.
A `BackButton` is never listed.
A button whose `pressBy` attribute another button shares, compared as the selector engine does (trimmed, lowercased, whitespace collapsed), is not listed, because a selector would match both.
The runner reads hittability only for the buttons that pass the other checks.
The runner never presses these controls.
With no entry, or when the read fails or hangs, the key is absent.
The daemon publishes the list as `details.dismissControls` and adds a selector to each entry: `role=button label="…"` or `role=button id="…"`. The `role=button` term limits the match to button elements, the set the runner checked.

The swipe is first resolved up to its contact. Then the settle window is a run of samples, with a
0.05 s pause between captures plus each capture's own time, that agree for the still span: a contact
allowance, a 0.25 s margin, and the slowest capture seen while settling, about 0.55 s on a simulator.
Expand Down
19 changes: 19 additions & 0 deletions contracts/fixtures/selector-text-normalization.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
[
{
"name": "surrounding spaces and a tab are trimmed, case is folded",
"input": " Cancel\t",
"normalized": "cancel"
},
{
"name": "a run of whitespace inside the text is one space",
"input": "Close\n\n Now",
"normalized": "close now"
},
{ "name": "no-break spaces count as whitespace", "input": " Back ", "normalized": "back" },
{
"name": "an icon name keeps its dots",
"input": "XMARK.circle.fill",
"normalized": "xmark.circle.fill"
},
{ "name": "whitespace only is empty", "input": " \t\n", "normalized": "" }
]
Loading
Loading