From 93dd75507dd982b710af81bca77f5534f584f70d Mon Sep 17 00:00:00 2001 From: dannyhw Date: Tue, 17 Nov 2026 11:42:13 +0000 Subject: [PATCH 1/2] feat(android): publish clickable, focusable, long-clickable and scrollable on snapshot nodes A snapshot node carries hittable, which is true for a node that is clickable, focusable or focused. A reader that audits what an accessibility service is offered needs those apart: a node a screen reader can land on but cannot activate is hittable, and so is one it can activate. The Android reader already parses clickable, focusable and scrollable; this publishes them on the node. The snapshot helper now writes long-clickable, with both answers so a capture from an older helper is not read as a node that offers no long click. --- .../snapshothelper/AccessibilityTreeXml.java | 5 +-- .../capture/runtime/snapshot-unchanged.ts | 4 +++ packages/kernel/src/snapshot.ts | 13 ++++++++ .../__tests__/snapshot-clickability.test.ts | 31 +++++++++++++++---- .../src/ui-hierarchy-builder.ts | 4 +++ .../platform-android/src/ui-hierarchy-node.ts | 2 ++ packages/platform-android/src/ui-hierarchy.ts | 4 +++ 7 files changed, 55 insertions(+), 8 deletions(-) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 334fcaf115..1ce5be840a 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -61,6 +61,9 @@ static void appendNode( appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); + // Both answers, like `selected` below: a helper older than this attribute reports nothing, and + // the host keeps that apart from a node that offers no long click. + appendAttribute(xml, "long-clickable", Boolean.toString(node.isLongClickable())); appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled())); appendTrueAttribute(xml, "focusable", node.isFocusable()); appendTrueAttribute(xml, "focused", node.isFocused()); @@ -188,8 +191,6 @@ private static boolean isHeading(AccessibilityNodeInfo node) { return Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && node.isHeading(); } - // Declared residue (agent-device #1832): long-clickable is not serialized. Adding it is a helper - // protocol change (new attribute + host parser + field on the wire node). private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); diff --git a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts index 7d293b61d8..b71f23dc82 100644 --- a/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts +++ b/packages/command-runtime/src/commands/capture/runtime/snapshot-unchanged.ts @@ -131,6 +131,10 @@ const PRESENTATION_SCALAR_FIELDS = { roleDescription: true, placeholder: true, hittable: true, + clickable: true, + focusable: true, + longClickable: true, + scrollable: true, bundleId: true, systemBar: true, appName: true, diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index e8dc06b0c0..53fc01e3bc 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -333,6 +333,19 @@ export type RawSnapshotNode = { /** UIKit `isUserInteractionEnabled`; absent means the producer did not read it, not false. */ userInteractionEnabled?: boolean; hittable?: boolean; + /** + * Android accessibility facts behind `hittable`, each on its own: `hittable` answers "can an + * agent drive this node" and folds a touch target and a focus target into one, while a reader + * that audits what an accessibility service is offered needs them apart. `clickable` is what a + * screen reader activates, `focusable` is what input focus can land on, `longClickable` is what + * it offers a long press on, and `scrollable` is a container announced as one. Absent means the + * producer did not read the fact, not false; `longClickable` is absent on a capture from an + * Android helper older than the attribute. + */ + clickable?: boolean; + focusable?: boolean; + longClickable?: boolean; + scrollable?: boolean; depth?: number; parentIndex?: number; pid?: number; diff --git a/packages/platform-android/src/__tests__/snapshot-clickability.test.ts b/packages/platform-android/src/__tests__/snapshot-clickability.test.ts index 63b979b7dd..e64926901a 100644 --- a/packages/platform-android/src/__tests__/snapshot-clickability.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-clickability.test.ts @@ -42,9 +42,9 @@ afterEach(() => { test('retains Android helper clickable facts from the same hierarchy projection', () => { const built = buildUiHierarchySnapshot( parseUiHierarchyTree(` - - - + + + `), undefined, { raw: true }, @@ -63,11 +63,27 @@ test('retains Android helper clickable facts from the same hierarchy projection' ], ); assert.equal(publicSnapshot.nodes[1]?.hittable, true); - assert.equal('clickable' in (publicSnapshot.nodes[1] ?? {}), false); + // The facts `hittable` folds together are published each on its own. A focusable node that is + // not clickable is hittable, and only the separate facts tell it from a touch target. + assert.deepEqual( + publicSnapshot.nodes.map((node) => [node.clickable, node.focusable, node.scrollable]), + [ + [true, false, false], + [false, true, false], + [false, false, true], + ], + ); + // Both answers when the capture reported one, and nothing when it did not: a helper older than + // the attribute is not read as "offers no long click". + assert.deepEqual( + publicSnapshot.nodes.map((node) => node.longClickable), + [true, false, undefined], + ); + assert.equal('longClickable' in (publicSnapshot.nodes[2] ?? {}), false); assert.equal(readSnapshotClickabilityEvidence(publicSnapshot), undefined); }); -test('snapshotAndroid captures the hierarchy once and retains clickability off-wire', async () => { +test('snapshotAndroid captures the hierarchy once and publishes the clickability it retains', async () => { let hierarchyCaptures = 0; const helperAdb: AndroidAdbExecutor = async (args) => { if (args.includes('--show-versioncode')) { @@ -119,7 +135,10 @@ test('snapshotAndroid captures the hierarchy once and retains clickability off-w [2, 2], ], ); - assert.equal(JSON.stringify(result).includes('clickable'), false); + assert.deepEqual( + result.nodes.map((node) => node.clickable), + [false, true, false], + ); assert.equal(JSON.stringify(result).includes('sourceIndexByNodeIndex'), false); }); diff --git a/packages/platform-android/src/ui-hierarchy-builder.ts b/packages/platform-android/src/ui-hierarchy-builder.ts index 2821885b39..bde92fdc5d 100644 --- a/packages/platform-android/src/ui-hierarchy-builder.ts +++ b/packages/platform-android/src/ui-hierarchy-builder.ts @@ -478,6 +478,10 @@ function createAndroidRawSnapshotNode( focused: node.focused, visibleToUser: node.visibleToUser, hittable: isAgentTarget(node) || undefined, + clickable: node.clickable, + focusable: node.focusable, + ...(node.longClickable === undefined ? {} : { longClickable: node.longClickable }), + scrollable: node.scrollable === true, depth: compactedAndroidNodeDepth(state.nodes, parentIndex), parentIndex, ...androidScrollActionHints(node, state.hidden), diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index 34197445dc..b41f18da24 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -37,6 +37,8 @@ export type AndroidUiHierarchy = { // encodings of one control opposite answers. clickable: boolean; focusable: boolean; + /** Whether the node offers a long click. Absent when the capture did not report it. */ + longClickable?: boolean; depth: number; parentIndex?: number; scrollable?: boolean; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index 7ef9b77d94..fc82b9998e 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -30,6 +30,8 @@ export type AndroidUiNodeMetadata = { bounds: string | null; rect?: Rect; clickable?: boolean; + /** Absent when the capture did not report it: a helper older than the attribute. */ + longClickable?: boolean; enabled?: boolean; visibleToUser?: boolean; drawingOrder?: number; @@ -166,6 +168,7 @@ function readNodeAttributes(node: string): Omit { className: getAttr('class'), bounds: getAttr('bounds'), clickable: boolAttr('clickable'), + ...optionalBoolAttr('longClickable', 'long-clickable'), enabled: boolAttr('enabled'), focusable: boolAttr('focusable'), focused: boolAttr('focused'), @@ -350,6 +353,7 @@ function normalizeAndroidUiHierarchyNode( focused: attrs.focused, visibleToUser: attrs.visibleToUser, clickable: attrs.clickable === true, + longClickable: attrs.longClickable, focusable: attrs.focusable === true, scrollable: attrs.scrollable, canScrollForward: attrs.canScrollForward, From d56b15a98d26aa71e6fcc90a68c5b9f31e4d77dc Mon Sep 17 00:00:00 2001 From: dannyhw Date: Tue, 17 Nov 2026 11:51:11 +0000 Subject: [PATCH 2/2] chore(android): drop the comments added with the snapshot facts --- .../agentdevice/snapshothelper/AccessibilityTreeXml.java | 2 -- packages/kernel/src/snapshot.ts | 9 --------- .../src/__tests__/snapshot-clickability.test.ts | 4 ---- packages/platform-android/src/ui-hierarchy-node.ts | 1 - packages/platform-android/src/ui-hierarchy.ts | 1 - 5 files changed, 17 deletions(-) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java index 1ce5be840a..31c3a11db1 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/AccessibilityTreeXml.java @@ -61,8 +61,6 @@ static void appendNode( appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); - // Both answers, like `selected` below: a helper older than this attribute reports nothing, and - // the host keeps that apart from a node that offers no long click. appendAttribute(xml, "long-clickable", Boolean.toString(node.isLongClickable())); appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled())); appendTrueAttribute(xml, "focusable", node.isFocusable()); diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index 53fc01e3bc..688a876360 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -333,15 +333,6 @@ export type RawSnapshotNode = { /** UIKit `isUserInteractionEnabled`; absent means the producer did not read it, not false. */ userInteractionEnabled?: boolean; hittable?: boolean; - /** - * Android accessibility facts behind `hittable`, each on its own: `hittable` answers "can an - * agent drive this node" and folds a touch target and a focus target into one, while a reader - * that audits what an accessibility service is offered needs them apart. `clickable` is what a - * screen reader activates, `focusable` is what input focus can land on, `longClickable` is what - * it offers a long press on, and `scrollable` is a container announced as one. Absent means the - * producer did not read the fact, not false; `longClickable` is absent on a capture from an - * Android helper older than the attribute. - */ clickable?: boolean; focusable?: boolean; longClickable?: boolean; diff --git a/packages/platform-android/src/__tests__/snapshot-clickability.test.ts b/packages/platform-android/src/__tests__/snapshot-clickability.test.ts index e64926901a..2fb76af056 100644 --- a/packages/platform-android/src/__tests__/snapshot-clickability.test.ts +++ b/packages/platform-android/src/__tests__/snapshot-clickability.test.ts @@ -63,8 +63,6 @@ test('retains Android helper clickable facts from the same hierarchy projection' ], ); assert.equal(publicSnapshot.nodes[1]?.hittable, true); - // The facts `hittable` folds together are published each on its own. A focusable node that is - // not clickable is hittable, and only the separate facts tell it from a touch target. assert.deepEqual( publicSnapshot.nodes.map((node) => [node.clickable, node.focusable, node.scrollable]), [ @@ -73,8 +71,6 @@ test('retains Android helper clickable facts from the same hierarchy projection' [false, false, true], ], ); - // Both answers when the capture reported one, and nothing when it did not: a helper older than - // the attribute is not read as "offers no long click". assert.deepEqual( publicSnapshot.nodes.map((node) => node.longClickable), [true, false, undefined], diff --git a/packages/platform-android/src/ui-hierarchy-node.ts b/packages/platform-android/src/ui-hierarchy-node.ts index b41f18da24..1c41ad673a 100644 --- a/packages/platform-android/src/ui-hierarchy-node.ts +++ b/packages/platform-android/src/ui-hierarchy-node.ts @@ -37,7 +37,6 @@ export type AndroidUiHierarchy = { // encodings of one control opposite answers. clickable: boolean; focusable: boolean; - /** Whether the node offers a long click. Absent when the capture did not report it. */ longClickable?: boolean; depth: number; parentIndex?: number; diff --git a/packages/platform-android/src/ui-hierarchy.ts b/packages/platform-android/src/ui-hierarchy.ts index fc82b9998e..308c0deb06 100644 --- a/packages/platform-android/src/ui-hierarchy.ts +++ b/packages/platform-android/src/ui-hierarchy.ts @@ -30,7 +30,6 @@ export type AndroidUiNodeMetadata = { bounds: string | null; rect?: Rect; clickable?: boolean; - /** Absent when the capture did not report it: a helper older than the attribute. */ longClickable?: boolean; enabled?: boolean; visibleToUser?: boolean;