diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 96b05b33a9..81d287b764 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -19,6 +19,7 @@ It is intentionally small, but each surface is dense with durable accessibility - `Automation lab`: long-press, alert-result, app-event, app-state, appearance, orientation, permission-recovery, log canaries, a flattened (`accessible={true}`) text input, and an Apple Pay sheet hosted in `com.apple.PassbookUIService` - `WebView accessibility`: a deterministic semantic fixture plus live websites with varied HTML for native accessibility snapshot verification - `Perf lab`: seeded JavaScript render problems for `react-devtools profile`, one per scenario (see [Perf lab](#perf-lab)) +- `Settle lab`: timed effects for `--settle` measurements (in-place counter, 500 and 800 ms delayed results, a 750 ms fading overlay, two-stage transitions with and without a spinner, and a toast that precedes navigation), reached by the `agent-device-test-app://settle-lab` deep link - `Heavy feed`: a Bluesky-shaped home feed whose accessibility tree is deeper than 65 levels (see [Heavy feed](#heavy-feed)) Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface. diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx index e452a411d2..38ca0c0e9f 100644 --- a/examples/test-app/app/_layout.tsx +++ b/examples/test-app/app/_layout.tsx @@ -31,6 +31,8 @@ function RootLayoutContent() { + + diff --git a/examples/test-app/app/settle-lab-result.tsx b/examples/test-app/app/settle-lab-result.tsx new file mode 100644 index 0000000000..d89fc10137 --- /dev/null +++ b/examples/test-app/app/settle-lab-result.tsx @@ -0,0 +1,8 @@ +import { useLocalSearchParams } from 'expo-router'; + +import { SettleLabResultScreen } from '../src/screens/SettleLabScreen'; + +export default function SettleLabResultRoute() { + const { run } = useLocalSearchParams<{ run?: string }>(); + return ; +} diff --git a/examples/test-app/app/settle-lab.tsx b/examples/test-app/app/settle-lab.tsx new file mode 100644 index 0000000000..47a8fa8625 --- /dev/null +++ b/examples/test-app/app/settle-lab.tsx @@ -0,0 +1,5 @@ +import { SettleLabScreen } from '../src/screens/SettleLabScreen'; + +export default function SettleLabRoute() { + return ; +} diff --git a/examples/test-app/src/screens/SettleLabScreen.tsx b/examples/test-app/src/screens/SettleLabScreen.tsx new file mode 100644 index 0000000000..d04beadccd --- /dev/null +++ b/examples/test-app/src/screens/SettleLabScreen.tsx @@ -0,0 +1,191 @@ +import { router } from 'expo-router'; +import { useEffect, useRef, useState } from 'react'; +import { ActivityIndicator, Animated, Pressable, StyleSheet, Text, View } from 'react-native'; + +/** + * Timed UI effects for measuring `--settle`: each button starts one effect whose final screen is + * known, so a settled tree can be checked against it. `settle-run` acknowledges each effect at once, + * the way an app shows a pending state, and `settle-status` names the final state of the last effect + * with its run number. + */ +const DELAYS_MS = [500, 800] as const; +const FADE_MS = 750; +const STAGE_MS = 600; +const NAVIGATE_AFTER_TOAST_MS = 700; + +export function SettleLabScreen() { + const [run, setRun] = useState(0); + const [count, setCount] = useState(0); + const [status, setStatus] = useState('idle'); + const [stage, setStage] = useState<'none' | 'spinner' | 'plain'>('none'); + const [overlayVisible, setOverlayVisible] = useState(false); + const [toast, setToast] = useState(null); + const opacity = useRef(new Animated.Value(1)).current; + const timers = useRef[]>([]); + + useEffect(() => () => timers.current.forEach(clearTimeout), []); + + const later = (ms: number, effect: () => void) => { + const handle = setTimeout(() => { + timers.current = timers.current.filter((pending) => pending !== handle); + effect(); + }, ms); + timers.current.push(handle); + }; + const nextRun = () => { + const next = run + 1; + setRun(next); + return next; + }; + + return ( + + Settle lab + + {status} + + + {`Count ${count}`} + + + {`Run ${run}`} + + setCount((c) => c + 1)} /> + {DELAYS_MS.map((delayMs) => ( + { + const id = nextRun(); + later(delayMs, () => setStatus(`Delayed ${delayMs} done ${id}`)); + }} + /> + ))} + { + const id = nextRun(); + opacity.setValue(1); + setOverlayVisible(true); + Animated.timing(opacity, { + toValue: 0, + duration: FADE_MS, + useNativeDriver: true, + }).start(() => { + setOverlayVisible(false); + setStatus(`Overlay gone ${id}`); + }); + }} + /> + { + const id = nextRun(); + setStage('spinner'); + later(STAGE_MS, () => { + setStage('none'); + setStatus(`Stage done ${id}`); + }); + }} + /> + { + const id = nextRun(); + setStage('plain'); + later(STAGE_MS, () => { + setStage('none'); + setStatus(`Stage done ${id}`); + }); + }} + /> + { + const id = nextRun(); + setToast('Saving'); + later(NAVIGATE_AFTER_TOAST_MS, () => { + setToast(null); + router.push({ pathname: '/settle-lab-result', params: { run: String(id) } }); + }); + }} + /> + {stage === 'spinner' ? ( + + + Working + + ) : null} + {stage === 'plain' ? ( + + Step 1 of 2 + + ) : null} + {toast ? ( + + {toast} + + ) : null} + {overlayVisible ? ( + + + Overlay + + + ) : null} + + ); +} + +export function SettleLabResultScreen({ run }: { run: string | undefined }) { + return ( + + {`Result ${run ?? ''}`} + router.back()} /> + + ); +} + +function LabButton(props: { id: string; label: string; onPress: () => void }) { + return ( + + {props.label} + + ); +} + +const styles = StyleSheet.create({ + frame: { flex: 1, gap: 8, padding: 24, paddingTop: 72, backgroundColor: '#fff' }, + title: { fontSize: 24, fontWeight: '700' }, + status: { fontSize: 16 }, + row: { flexDirection: 'row', gap: 8, alignItems: 'center' }, + button: { backgroundColor: '#e8eefc', borderRadius: 8, padding: 12 }, + buttonLabel: { fontSize: 16 }, + toast: { + position: 'absolute', + bottom: 48, + left: 24, + right: 24, + backgroundColor: '#222', + borderRadius: 8, + padding: 12, + }, + toastLabel: { color: '#fff', fontSize: 16 }, + overlay: { + ...StyleSheet.absoluteFill, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: 'rgba(0,0,0,0.5)', + }, + overlayButton: { backgroundColor: '#333', borderRadius: 8, padding: 24 }, +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts index ea6c7f0962..4d610652b3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts @@ -60,6 +60,8 @@ export function welcomeSnapshot(): SnapshotState { export function createSettleDevice(params: { stored: SnapshotState; + /** The ADR 0014 ref frame, when the scenario issued one apart from the stored observation. */ + refFrame?: SnapshotState; captureSnapshot: () => Promise | BackendSnapshotResult; tap?: () => Promise>; clock?: ReturnType; @@ -78,6 +80,7 @@ export function createSettleDevice(params: { { name: 'default', snapshot: params.stored, + ...(params.refFrame ? { refFrameSnapshot: params.refFrame } : {}), ...(params.appBundleId ? { appBundleId: params.appBundleId } : {}), }, ]), diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts new file mode 100644 index 0000000000..cb62137296 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts @@ -0,0 +1,314 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotProvenance, SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref, selector } from './selector-read-utils.ts'; +import { createSettleDevice, welcomeSnapshot } from './__tests__/settle-device-fixtures.ts'; +import { + elementCatalogTabSnapshot, + elementSettingsTabSnapshot, +} from './stable-capture.fixtures.ts'; + +// The internal adaptive confirmation prototype: a capture that differs from a fresh pre-action +// baseline settles once one short confirmation agrees with it; every escalation keeps the quiet +// window. Each case runs the same capture sequence with the switch on and off, so the difference +// is the rule's and not the fixture's. Budgets run on the injected clock; captures cost nothing. + +const DEFAULT_QUIET_MS = 500; + +/** The welcome screen after a press relabels its heading in place: no broad replacement. */ +function relabeled(label = 'Welcome back!'): SnapshotState { + const base = welcomeSnapshot(); + return { + ...base, + nodes: base.nodes.map((node) => (node.index === 0 ? { ...node, label } : node)), + }; +} + +function withProvenance

(state: SnapshotState, provenance: P) { + return { ...state, ...provenance }; +} + +function withSpinner(snapshot: SnapshotState): SnapshotState { + return makeSnapshotState([ + ...snapshot.nodes, + { + index: snapshot.nodes.length, + depth: 1, + parentIndex: 0, + type: 'ActivityIndicator', + label: 'In progress', + rect: { x: 10, y: 400, width: 20, height: 20 }, + }, + ]); +} + +/** + * An @ref press on the stored tree, whose loop captures follow `loop` (the last one repeats). The + * native ref route captures nothing before the action, so the stored tree is the baseline, fresh + * here because the daemon reports that observation current. + */ +async function press(params: { + stored?: SnapshotState; + loop: SnapshotState[]; + adaptive: boolean; + quietMs?: number; + target?: Parameters['interactions']['press']>[0]; +}) { + let captures = 0; + const device = createSettleDevice({ + stored: params.stored ?? welcomeSnapshot(), + captureSnapshot: () => { + captures += 1; + return { snapshot: params.loop[Math.min(captures, params.loop.length) - 1]! }; + }, + }); + const result = await device.interactions.press(params.target ?? ref('@e2'), { + session: 'default', + settle: { + ...(params.adaptive ? { earlyExit: 'adaptive' as const, observationCurrent: true } : {}), + ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), + }, + }); + assert.ok(result.settle); + return result.settle; +} + +test('a changed capture that the next capture confirms settles before the quiet window', async () => { + const loop = [relabeled(), relabeled()]; + + const adaptive = await press({ loop, adaptive: true }); + const quiet = await press({ loop, adaptive: false }); + + assert.equal(adaptive.settled, true); + assert.equal(adaptive.captures, 2); + assert.ok(adaptive.waitedMs < 200, `adaptive held ${adaptive.waitedMs}ms`); + // The run reports the span the confirming pair held, not the window it never waited out. + assert.equal(adaptive.quietMs, adaptive.waitedMs); + assert.ok(quiet.quietMs >= DEFAULT_QUIET_MS); + assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!'))); + assert.equal(quiet.settled, true); + assert.ok(quiet.waitedMs >= DEFAULT_QUIET_MS, `quiet window held ${quiet.waitedMs}ms`); +}); + +test('a confirmation that disagrees falls back to the full quiet window', async () => { + const adaptive = await press({ + loop: [relabeled('Loading'), relabeled('Welcome back!')], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); + assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!'))); +}); + +test('a capture that still matches the baseline keeps the quiet window', async () => { + const adaptive = await press({ loop: [welcomeSnapshot()], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a change that arrives after an unchanged capture is confirmed on its own', async () => { + const adaptive = await press({ + loop: [welcomeSnapshot(), relabeled(), relabeled()], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.equal(adaptive.captures, 3); + assert.ok(adaptive.waitedMs < DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a busy indicator on the changed screen keeps the quiet window', async () => { + const loop = [withSpinner(relabeled())]; + + const adaptive = await press({ loop, adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a confirmation from another producer keeps the quiet window', async () => { + const adaptive = await press({ + loop: [ + withProvenance(relabeled(), { backend: 'xctest', producer: 'apple-runner' }), + withProvenance(relabeled(), { backend: 'xctest', producer: 'appium-source' }), + ], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a presented surface the baseline did not show keeps the quiet window', async () => { + const adaptive = await press({ + stored: { ...welcomeSnapshot(), presentedSurface: 'absent' }, + loop: [{ ...relabeled(), presentedSurface: 'presented' }], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a broad replacement owed the confirmation floor keeps the floor', async () => { + const settings = elementSettingsTabSnapshot(); + + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [settings], + adaptive: true, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= 1_500, `adaptive lost the floor: ${adaptive.waitedMs}ms`); +}); + +test('a broad replacement under a requested window above the floor keeps that window', async () => { + // The floor raises nothing here, but the run still proved a broad replacement. + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [elementSettingsTabSnapshot()], + adaptive: true, + quietMs: 2_000, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok( + adaptive.waitedMs >= 2_000, + `adaptive lost the requested window: ${adaptive.waitedMs}ms`, + ); +}); + +test('a point press whose baseline is the stored tree keeps the quiet window', async () => { + // The pre-action evidence capture fails, so the baseline falls back to the stored tree, which may + // predate a side effect: a change against it does not prove the action's effect. + let captures = 0; + const device = createSettleDevice({ + stored: welcomeSnapshot(), + captureSnapshot: () => { + captures += 1; + if (captures === 1) throw new Error('capture unavailable'); + return { snapshot: relabeled() }; + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 50, y: 100 }, + { session: 'default', settle: { earlyExit: 'adaptive' } }, + ); + + assert.equal(result.settle?.settled, true); + assert.ok(result.settle.waitedMs >= DEFAULT_QUIET_MS, `held ${result.settle.waitedMs}ms`); +}); + +test('a capture that holds the baseline beside a new screen keeps the quiet window', async () => { + // A navigation transition can publish both screens at once and hold that union across captures. + const base = welcomeSnapshot(); + const union = makeSnapshotState([ + ...base.nodes, + ...['Detail', 'Buy', 'Share'].map((label, offset) => ({ + index: base.nodes.length + offset, + depth: 1, + parentIndex: 0, + type: 'Button', + label, + rect: { x: 200, y: 20 + offset * 60, width: 100, height: 40 }, + })), + ]); + + const adaptive = await press({ loop: [union], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +for (const [label, observationCurrent, early] of [ + ['a current observation is a fresh baseline', true, true], + ['a frame the daemon reports outdated keeps the quiet window', false, false], +] as const) { + test(`a selector press read in another view than the frame: ${label}`, async () => { + // The resolution capture carries no presentation, the frame is interactive-only, so the settle + // baseline is the frame itself and only the daemon can say whether it is current. + const frame = { ...welcomeSnapshot(), presentation: { interactiveOnly: true } }; + let captures = 0; + const device = createSettleDevice({ + stored: frame, + refFrame: frame, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? welcomeSnapshot() : relabeled() }; + }, + }); + + const result = await device.interactions.press(selector('label=Next'), { + session: 'default', + settle: { earlyExit: 'adaptive', observationCurrent }, + }); + + assert.equal(result.settle?.settled, true); + assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`); + }); +} + +for (const [label, observationCurrent, early] of [ + ['a current observation is a fresh baseline', true, true], + ['a frame the daemon reports outdated keeps the quiet window', false, false], +] as const) { + test(`an @ref press on the stored frame: ${label}`, async () => { + // The ref route resolves against the stored frame and captures nothing before the action, so the + // frame is the baseline. After `wait text` (or an Android freshness read) the frame stays active + // but no longer describes the screen, and only the daemon knows that. + const frame = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: frame, + refFrame: frame, + captureSnapshot: () => { + captures += 1; + return { snapshot: relabeled() }; + }, + }); + + const result = await device.interactions.press(ref('@e2'), { + session: 'default', + settle: { earlyExit: 'adaptive', observationCurrent }, + }); + + assert.equal(captures > 0, true); + assert.equal(result.settle?.settled, true); + assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`); + }); +} + +test('a capture on an iOS system surface the baseline did not show keeps the quiet window', async () => { + // An app action can raise an in-place system surface (a web sign-in sheet, #2438); two equal + // captures of it are not the app settling. + const sheet = { ...relabeled(), iosSystemSurfaceBundleId: 'com.apple.SafariViewService' }; + + const adaptive = await press({ loop: [sheet], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a rootless broad replacement the latch could not judge yet keeps the quiet window', async () => { + // A rootless read after a non-presented baseline fails the latch gate, so the floor stays an open + // question; a later rooted capture could still confirm it, so two rootless reads must not settle. + const rootless = elementSettingsTabSnapshot(); + rootless.nodes = rootless.nodes.filter((node) => node.type !== 'Application'); + + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [rootless], + adaptive: true, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts new file mode 100644 index 0000000000..18030d75af --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -0,0 +1,157 @@ +import { isBusyIndicatorNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; +import { + sharedIdentityCount, + stableCaptureSignal, + stableCaptureSignalsEqual, + type StableCaptureSignal, +} from './stable-capture-signal.ts'; + +/** + * Internal prototype (`SettleParams.earlyExit: 'adaptive'`): a capture that differs from a fresh + * pre-action baseline, and that one short confirmation capture agrees with, settles before the + * quiet window. Every other case keeps the quiet window: no capture has differed yet (a no-op, a + * toggle that has not rendered, a late effect), the confirmation disagrees, a busy indicator is on + * screen, the producer or the presented surface changed, the broad-replacement floor is in force, + * the capture still holds the baseline screen beside a new one, or the baseline may predate a side + * effect. One failed check disarms the rule for the rest of the run. + * + * Two equal captures ~110 ms apart cannot exclude a later stage nothing marks: a toast standing in + * for navigation, a plateau between transition stages, a late value write. That is the trade this + * prototype measures. + */ +const ADAPTIVE_CONFIRMATION_GAP_MS = 110; + +export type AdaptiveEarlyExit = ReturnType; + +export function createAdaptiveConfirmation(params: { + baseline: SurfaceScopedNodes | undefined; + baselineFresh: boolean; + appBundleId: string | undefined; +}) { + const { baseline, appBundleId } = params; + let armed = params.baselineFresh && baseline !== undefined && baseline.nodes.length > 0; + let candidate: { signal: StableCaptureSignal; snapshot: SnapshotState; atMs: number } | undefined; + const disarm = () => { + armed = false; + candidate = undefined; + }; + return { + /** + * Folds one capture in. `settle`: this capture confirmed the previous one, and the pair held + * still for `heldQuietMs`. `confirm`: take the next capture after the short gap. `none`: the + * quiet window decides. + */ + observe(input: { + snapshot: SnapshotState; + signal: StableCaptureSignal; + nowMs: number; + /** The run owes, or may yet owe, the broad-replacement floor for this capture. */ + floorInForce: boolean; + }): { kind: 'settle'; heldQuietMs: number } | { kind: 'confirm' } | { kind: 'none' } { + if (!armed || !baseline) return { kind: 'none' }; + const { snapshot, signal } = input; + if ( + input.floorInForce || + hasBusyIndicator(snapshot) || + isTransitionalUnion(baseline, snapshot, appBundleId) || + presentedSurfaceChanged(baseline, snapshot) || + surfaceBundleChanged(baseline, snapshot) + ) { + disarm(); + return { kind: 'none' }; + } + if (candidate) { + const confirmed = + sameProvenance(candidate.snapshot, snapshot) && + stableCaptureSignalsEqual(candidate.signal, signal); + const heldQuietMs = input.nowMs - candidate.atMs; + disarm(); + return confirmed ? { kind: 'settle', heldQuietMs } : { kind: 'none' }; + } + if (!differsFromBaseline(baseline, snapshot, appBundleId)) return { kind: 'none' }; + candidate = { signal, snapshot, atMs: input.nowMs }; + return { kind: 'confirm' }; + }, + /** A private-AX recovery re-read the screen through another backend. */ + disarm, + /** The wait before the confirming capture. */ + confirmGapMs: ADAPTIVE_CONFIRMATION_GAP_MS, + }; +} + +/** + * A navigation transition can publish both screens at once, and that union can hold still across + * two captures. A capture that keeps most of the baseline while adding half as much again is that + * union, not a settled screen. + */ +const UNION_RETAINED_RATIO = 0.8; +const UNION_GROWTH_RATIO = 0.5; + +function isTransitionalUnion( + baseline: SurfaceScopedNodes, + snapshot: SnapshotState, + appBundleId: string | undefined, +): boolean { + const before = projectedSignal(snapshot, baseline.nodes, appBundleId); + const after = projectedSignal(snapshot, snapshot.nodes, appBundleId); + if (before.nodes.length === 0) return false; + const shared = sharedIdentityCount(before, after); + return ( + shared / before.nodes.length >= UNION_RETAINED_RATIO && + (after.nodes.length - shared) / before.nodes.length >= UNION_GROWTH_RATIO + ); +} + +/** + * One side of a baseline comparison: both sides take the capture's provenance so visibility + * filtering applies alike, and both drop the keyboard chrome the settle diff collapses. + */ +function projectedSignal( + snapshot: SnapshotState, + nodes: SnapshotState['nodes'], + appBundleId: string | undefined, +): StableCaptureSignal { + return stableCaptureSignal({ ...snapshot, nodes: withoutSettleChrome(nodes, appBundleId) }); +} + +function differsFromBaseline( + baseline: SurfaceScopedNodes, + snapshot: SnapshotState, + appBundleId: string | undefined, +): boolean { + // A keyboard alone is not the action's effect: the projection drops it. + return !stableCaptureSignalsEqual( + projectedSignal(snapshot, baseline.nodes, appBundleId), + projectedSignal(snapshot, snapshot.nodes, appBundleId), + ); +} + +function hasBusyIndicator(snapshot: SnapshotState): boolean { + return snapshot.nodes.some(isBusyIndicatorNode); +} + +function presentedSurfaceChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean { + const before = baseline.presentedSurface; + const after = snapshot.presentedSurface; + return before !== undefined && after !== undefined && before !== after; +} + +/** + * The capture reads another surface than the baseline: an in-place iOS system surface (a web sign-in + * sheet) arriving over the app, or leaving it (#2438). An app capture names none. + */ +function surfaceBundleChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean { + return baseline.surfaceBundleId !== snapshot.iosSystemSurfaceBundleId; +} + +function sameProvenance(left: SnapshotState, right: SnapshotState): boolean { + return ( + left.producer === right.producer && + left.backend === right.backend && + left.presentedSurface === right.presentedSurface && + left.iosSystemSurfaceBundleId === right.iosSystemSurfaceBundleId + ); +} diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index dd0c64a38d..8aee0cb71c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -92,7 +92,7 @@ export async function settleAfterInteraction( ): Promise { return await settleAfterAction(runtime, options, { ...params, - baseline: await resolveSettleBaseline(runtime, options, params.resolved), + ...(await resolveSettleBaseline(runtime, options, params.resolved, params.observationCurrent)), actionPoint: params.resolved.point, }); } @@ -101,6 +101,11 @@ export type SettleObservationCommandOptions = CommandContext & SettleParams & { /** The pre-action tree the settled diff is taken against, and the surface it describes. */ baseline: SurfaceScopedNodes; + /** + * Whether `baseline` describes the screen right before this action. Absent means it may predate + * a side effect, so the adaptive confirmation never treats a change against it as the action's. + */ + baselineFresh?: boolean; }; /** @@ -133,7 +138,11 @@ export const settleObservationCommand: RuntimeCommand< async function settleAfterAction( runtime: AgentDeviceRuntime, options: CommandContext, - params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point }, + params: SettleParams & { + baseline: SurfaceScopedNodes; + baselineFresh?: boolean; + actionPoint?: Point; + }, ): Promise { // The loop resolves an unrequested window from the capture that pays it, so the window is only // known once one is taken. `base` carries the general default for the pre-capture shape the @@ -154,6 +163,9 @@ async function settleAfterAction( timeoutMs, resetBudgetOnPrivateAxRecovery: true, broadTransitionBaseline: params.baseline, + ...(params.earlyExit === 'adaptive' + ? { adaptiveConfirmation: { baselineFresh: params.baselineFresh === true } } + : {}), }); if (outcome.failure) return settleUnavailable(base, outcome, outcome.failure.error); return await readSettledOutcome(runtime, options, params, base, outcome); @@ -237,9 +249,11 @@ async function resolveSettleBaseline( runtime: AgentDeviceRuntime, options: CommandContext, resolved: ResolvedInteractionTarget, -): Promise { + observationCurrent: boolean | undefined, +): Promise<{ baseline: SurfaceScopedNodes; baselineFresh: boolean }> { const session = await runtime.sessions.get(options.session ?? 'default'); const frame = surfaceScopedNodes(session?.refFrameSnapshot); + const refFrame = resolved.kind === 'ref' ? frame : undefined; // A ref is authorized against the stored ref frame. Keep that visible presentation as the // transition baseline: a best-effort evidence recapture can recover through private AX and see // covered background controls that were not actionable when the ref was issued. @@ -247,16 +261,22 @@ async function resolveSettleBaseline( // authoritative ref frame, so reuse it rather than silently turning a missing optional field // into an empty transition/diff baseline. Fall back to the latest observation for point targets // and pre-frame sessions. A candidate read in another view than the frame is skipped (#4486). - return postActionBaseline({ + const baseline = await postActionBaseline({ frame: withoutPresentedSurface(frame), candidates: [ - resolved.kind === 'ref' ? frame : undefined, + refFrame, resolved.preAction, session?.refFrameSnapshot ? undefined : withoutPresentedSurface(surfaceScopedNodes(session?.snapshot)), ], }); + // Only a capture this action took (a selector resolution or a point's evidence read) proves the + // screen before it. A ref resolves against the stored frame, so its `preAction` is that frame too; + // the frame and the stored tree are fresh only while the daemon reports the observation current. + const actionCapture = resolved.kind !== 'ref' && baseline === resolved.preAction; + const baselineFresh = baseline.nodes.length > 0 && (actionCapture || observationCurrent === true); + return { baseline, baselineFresh }; } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts index fcc6a02c47..e2a37cd415 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts @@ -53,6 +53,11 @@ export function stableCaptureSignalsHaveBroadReplacement( // semantic replacement. Both signals carry the same chrome-free visible projection the settle // diff collapses, and the transition baseline is only armed for XCTest sessions. if (!left || left.nodes.length === 0 || right.nodes.length === 0) return false; + return sharedIdentityCount(left, right) / Math.max(left.nodes.length, right.nodes.length) < 0.5; +} + +/** How many nodes the two signals share by semantic identity, each node matched at most once. */ +export function sharedIdentityCount(left: StableCaptureSignal, right: StableCaptureSignal): number { const remainingIdentities = new Map(); for (const node of left.nodes) { remainingIdentities.set(node.identity, (remainingIdentities.get(node.identity) ?? 0) + 1); @@ -64,7 +69,7 @@ export function stableCaptureSignalsHaveBroadReplacement( sharedNodes += 1; remainingIdentities.set(node.identity, remaining - 1); } - return sharedNodes / Math.max(left.nodes.length, right.nodes.length) < 0.5; + return sharedNodes; } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 0060264cd2..de7ed7665e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -25,6 +25,7 @@ import { } from './stable-capture-signal.ts'; import { preferredSnapshotBackendForVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; +import type { AdaptiveEarlyExit } from './settle-adaptive-confirmation.ts'; /** * The quiet-window stable-capture loop shared by `wait stable` and the @@ -84,7 +85,8 @@ export type StableCaptureLoopResult = { /** * The quiet window the run was holding when it returned: the caller's, or the default resolved * from the last capture that paid it. A run that completed no capture never priced a window, so - * it reports the general default it started from. + * it reports the general default it started from. A run the adaptive early exit settled reports + * the span its confirming pair actually held instead. */ quietMs: number; /** The most recent completed capture, settled or not. */ @@ -110,6 +112,11 @@ export async function runStableCaptureLoop( * captures in its presentation, so the settled tree and the baseline share one view (#4486). */ broadTransitionBaseline?: SurfaceScopedNodes; + /** + * Internal prototype: settle on a baseline-differing capture that one short confirmation agrees + * with (see settle-adaptive-confirmation.ts). Absent keeps the quiet window alone. + */ + adaptiveConfirmation?: { baselineFresh: boolean }; }, ): Promise { const { timeoutMs } = params; @@ -135,6 +142,7 @@ export async function runStableCaptureLoop( // loop recovers on private AX, the rest of the loop stays on it. let preferredBackend: SnapshotPreferredBackend | undefined; const pacing = createStableCapturePacing(start); + const adaptive = await loadAdaptiveEarlyExit(params, session?.appBundleId); let captures = 0; let lastCapture: CapturedSnapshot | undefined; const result = ( @@ -159,14 +167,7 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), - params.broadTransitionBaseline - ? postActionPresentation(params.broadTransitionBaseline) - : { - ...postActionPresentation(undefined), - depth: options.depth, - scope: options.scope, - raw: options.raw, - }, + stableCapturePresentation(params.broadTransitionBaseline, options), preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); @@ -199,17 +200,110 @@ export async function runStableCaptureLoop( if (recovered) { privateAxRecoveryBudgetReset = true; deadlineMs = recovered.deadlineMs; + adaptive?.disarm(); } const step = recovered ? { settled: false, delayMs: recovered.delayMs } - : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures }); - if (step.settled) return result(true, false, nowMs - start); + : observeStableCapture({ + pacing, + adaptive, + confirmation, + snapshot: capture.snapshot, + signal, + window, + nowMs, + deadlineMs, + captures, + }); + if (step.settled) { + quietMs = step.heldQuietMs ?? quietMs; + return result(true, false, nowMs - start); + } if (step.delayMs <= 0) break; await sleep(runtime, step.delayMs); } return result(false, false, now(runtime) - start); } +/** + * The view every capture of the run reads: the baseline's, so the settled tree and the baseline + * compare (#4486), or without one the interactive view the caller's options narrow. + */ +function stableCapturePresentation( + baseline: SurfaceScopedNodes | undefined, + options: SelectorSnapshotOptions, +): SnapshotPresentation { + if (baseline) return postActionPresentation(baseline); + return { + ...postActionPresentation(undefined), + depth: options.depth, + scope: options.scope, + raw: options.raw, + }; +} + +/** + * The internal adaptive early exit, loaded only when a settle asks for it so the default path keeps + * its import closure. + */ +async function loadAdaptiveEarlyExit( + params: { + broadTransitionBaseline?: SurfaceScopedNodes; + adaptiveConfirmation?: { baselineFresh: boolean }; + }, + appBundleId: string | undefined, +): Promise { + if (!params.adaptiveConfirmation) return undefined; + const { createAdaptiveConfirmation } = await import('./settle-adaptive-confirmation.ts'); + return createAdaptiveConfirmation({ + baseline: params.broadTransitionBaseline, + baselineFresh: params.adaptiveConfirmation.baselineFresh, + appBundleId, + }); +} + +/** + * Folds one capture into the quiet window and, when the adaptive early exit is armed, into its + * short confirmation: settled, or how long to wait before the next capture (<= 0 ends the run). + */ +function observeStableCapture(params: { + pacing: StableCapturePacing; + adaptive: AdaptiveEarlyExit | undefined; + confirmation: StableCaptureConfirmation; + snapshot: SnapshotState; + signal: StableCaptureSignal; + window: ReturnType; + nowMs: number; + deadlineMs: number; + captures: number; +}): { settled: boolean; delayMs: number; heldQuietMs?: number } { + const { window, adaptive } = params; + const step = params.pacing.observe({ + signal: params.signal, + nodeCount: params.snapshot.nodes.length, + nowMs: params.nowMs, + requiredQuietMs: window.requiredQuietMs, + pollMs: window.pollMs, + deadlineMs: params.deadlineMs, + captures: params.captures, + }); + if (step.settled || !adaptive) return step; + const early = adaptive.observe({ + snapshot: params.snapshot, + signal: params.signal, + nowMs: params.nowMs, + floorInForce: + window.floorConfirmed || params.confirmation.unresolvedReplacement(params.snapshot), + }); + if (early.kind === 'settle') { + return { settled: true, delayMs: 0, heldQuietMs: early.heldQuietMs }; + } + if (early.kind === 'confirm' && step.delayMs > 0) { + return { settled: false, delayMs: Math.min(step.delayMs, adaptive.confirmGapMs) }; + } + return step; +} + /** * The quiet window one capture prices, the window it must actually hold once the broad-transition * confirmation has had its say, and the cadence that can measure that. Sampling an extended wait on @@ -219,10 +313,15 @@ function stableCaptureWindow( snapshot: SnapshotState, requestedQuietMs: number | undefined, confirmation: StableCaptureConfirmation, -): { quietMs: number; requiredQuietMs: number; pollMs: number } { +): { quietMs: number; requiredQuietMs: number; pollMs: number; floorConfirmed: boolean } { const quietMs = requestedQuietMs ?? defaultQuietMsForCapture(snapshot); const requiredQuietMs = confirmation.requiredQuietMs(snapshot, quietMs); - return { quietMs, requiredQuietMs, pollMs: stableCapturePollMs(requiredQuietMs) }; + return { + quietMs, + requiredQuietMs, + pollMs: stableCapturePollMs(requiredQuietMs), + floorConfirmed: confirmation.confirmed, + }; } /** @@ -404,6 +503,19 @@ function broadTransitionConfirmation(params: { ? currentQuietMs : Math.max(currentQuietMs, confirmedQuietMs); }, + get confirmed() { + return confirmedQuietMs !== undefined; + }, + unresolvedReplacement(snapshot) { + // The gate has refused every capture so far, so the latch has not ruled; a later capture it + // admits could still confirm the floor. Judge this capture against the baseline directly. + if (!baseline || transitionBaseline !== undefined) return false; + if (!isBoundIosXCTestCapture(snapshot.backend, platform)) return false; + return stableCaptureSignalsHaveBroadReplacement( + stableCaptureComparisonSignal(snapshot, baseline.nodes, appBundleId), + stableCaptureComparisonSignal(snapshot, snapshot.nodes, appBundleId), + ); + }, }; } @@ -413,6 +525,17 @@ type StableCaptureConfirmation = { snapshot: Parameters[0], currentQuietMs: number, ) => number; + /** + * A capture proved the broad replacement, so the floor holds for the rest of the run, whether or + * not it raised the window: a requested window above the floor still owes its full span. + */ + readonly confirmed: boolean; + /** + * The latch is armed but no capture has passed its gate yet (a rootless read after a + * non-presented baseline), and this capture already replaces the baseline broadly. Asked only by + * the adaptive early exit, which must not settle before a later capture could confirm the floor. + */ + unresolvedReplacement: (snapshot: Parameters[0]) => boolean; }; /** diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 85e86560bc..03c4c1cfe2 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -300,6 +300,8 @@ export type DaemonSettleObservationOptions = CommandContext & Readonly<{ /** The pre-action tree the settled diff is taken against, and the surface it describes. */ baseline: SurfaceScopedNodes; + /** Whether `baseline` describes the screen right before this action (adaptive settle only). */ + baselineFresh?: boolean; }>; /** diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index edd81f0787..97d6198c4d 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -301,6 +301,18 @@ export type SettleTailEntry = { export type SettleParams = { quietMs?: number; timeoutMs?: number; + /** + * Internal prototype switch, never a CLI flag: 'adaptive' lets a baseline-differing capture that one + * short confirmation agrees with settle before the quiet window. The daemon sets it only when the + * host environment opts in (AGENT_DEVICE_SETTLE_ADAPTIVE=1). + */ + earlyExit?: 'adaptive'; + /** + * With `earlyExit`: when the action began, the session's ref frame was active, its tree was the + * newest observation, and no native read had outdated it. A baseline this action did not capture + * itself (the ref frame or the stored tree) is fresh only then. + */ + observationCurrent?: boolean; }; /** @@ -319,6 +331,10 @@ export type SettleObservation = { settled: boolean; waitedMs: number; captures: number; + /** + * The quiet window the settle held: the requested or producer default window, or, when the + * internal adaptive early exit settled the run, the span its confirming pair actually held. + */ quietMs: number; timeoutMs: number; /** diff --git a/packages/daemon/src/__tests__/support/store-factory.ts b/packages/daemon/src/__tests__/support/store-factory.ts index cf2efaa757..5ea996ebb8 100644 --- a/packages/daemon/src/__tests__/support/store-factory.ts +++ b/packages/daemon/src/__tests__/support/store-factory.ts @@ -1,6 +1,10 @@ import path from 'node:path'; import { SessionStore } from '../../core/session-store.ts'; -import { createLiveTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + createLiveTestHostEnvironment, + createTestHostEnvironment, + mkdtempForTestSync, +} from '@agent-device/test-kit'; import type { SessionRef, SessionState } from '../../core/session-state.ts'; export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore { @@ -8,6 +12,16 @@ export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore { return new SessionStore(path.join(tempRoot, 'sessions'), createLiveTestHostEnvironment()); } +/** A store whose daemon host environment adds `env`, such as an internal opt-in switch. */ +export function makeSessionStoreWithEnv(env: Record): SessionStore { + const tempRoot = mkdtempForTestSync('agent-device-test-'); + const host = createTestHostEnvironment(); + return new SessionStore(path.join(tempRoot, 'sessions'), { + ...host, + env: { ...host.env, ...env }, + }); +} + export function makeStoredSessionRef(session: SessionState, address = session.name): SessionRef { return makeSessionStore().publish(address, session); } diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 1ef7baa1ca..807fc0dbb4 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -3,7 +3,7 @@ import type { SettleObservation, SurfaceScopedNodes } from '@agent-device/contra import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime'; import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; -import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { formatRole, isBusyIndicatorNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { @@ -290,7 +290,7 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { const reading = node.label?.trim() || node.value?.trim(); if (!reading) continue; const role = formatRole(node.type ?? ''); - if (BUSY_INDICATOR_ROLES.has(role)) continue; + if (isBusyIndicatorNode(node)) continue; if (role !== 'text') return true; texts.add(reading); if (texts.size > 1) return true; @@ -298,11 +298,6 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { return false; } -const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ - 'activity-indicator', - 'progress-indicator', -]); - function clearSurfaceStabilization(session: SessionState | undefined): void { if (session?.deferredSurface?.kind !== 'stabilize') return; session.deferredSurface = undefined; diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts index f301e83e36..37ad70a295 100644 --- a/packages/daemon/src/execution/generic-settle.test.ts +++ b/packages/daemon/src/execution/generic-settle.test.ts @@ -7,7 +7,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { IosSystemSurfaceProvenance } from '@agent-device/contracts/ios-system-surface'; import { makeIosSession } from '../__tests__/support/session-factories.ts'; -import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { makeSessionStore, makeSessionStoreWithEnv } from '../__tests__/support/store-factory.ts'; import { activateCompleteRefFrame, markSessionSnapshotOutdated, @@ -712,3 +712,75 @@ for (const change of ['rebuild', 'retire'] as const) { } }); } + +const ADAPTIVE_ENV = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }; + +test('back --settle with the adaptive switch arms a current stored tree as a fresh baseline', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-current'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]).toMatchObject({ + quietMs: 25, + timeoutMs: 2_000, + earlyExit: 'adaptive', + observationCurrent: true, + baselineFresh: true, + }); +}); + +test('back --settle with the adaptive switch does not trust a stored tree a native read outdated', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-outdated'; + const session = seedSession(sessionName, sessionStore); + markSessionSnapshotOutdated(session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]).toMatchObject({ + earlyExit: 'adaptive', + observationCurrent: false, + baselineFresh: false, + }); +}); + +test('back --settle with the adaptive switch does not trust a baseline the frame answered', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-frame'; + // The frame was read interactive-only; a later full read replaced the stored tree, so the frame + // answers the baseline and a newer observation exists. + const session = makeIosSession(sessionName); + setSessionSnapshot( + session, + buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES }), { snapshotInteractiveOnly: true }), + ); + activateCompleteRefFrame(session); + setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {})); + sessionStore.publish(sessionName, session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]?.baseline.presentation?.interactiveOnly).toBe(true); + expect(settleObservationCalls[0]).toMatchObject({ + observationCurrent: false, + baselineFresh: false, + }); +}); + +test('back --settle without the adaptive switch forwards no early exit', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-adaptive-off'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]?.earlyExit).toBeUndefined(); + expect(settleObservationCalls[0]?.observationCurrent).toBeUndefined(); + expect(settleObservationCalls[0]?.baselineFresh).toBe(false); +}); diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index f2accd1537..3d9cfd889c 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -90,16 +90,22 @@ export async function planGenericSettleObservation( if (!commandSupportsSettleObservation(params.command)) return {}; const invalidSettleFlags = settleFlagGuardResponse(params.command, params.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; - const settle = readSettleRequest(params.flags); + const settle = readSettleRequest(params.flags, { + env: params.sessionStore.host.env, + session: params.session, + }); if (!settle) return {}; const { presentedSurface: _unverified, ...frame } = surfaceScopedNodes( refFrameTree(params.session), ); - const baseline = await postActionBaseline({ - frame, - candidates: [storedTreeBaseline(params.session.snapshot)], - }); - return { observe: async () => await observeSettled(params, settle, baseline) }; + const stored = storedTreeBaseline(params.session.snapshot); + const baseline = await postActionBaseline({ frame, candidates: [stored] }); + // The generic route captures nothing before the action, so the stored tree is fresh only while the + // daemon reports the session observation current. + const baselineFresh = baseline === stored && settle.observationCurrent === true; + return { + observe: async () => await observeSettled(params, { ...settle, baselineFresh }, baseline), + }; } /** @@ -124,7 +130,7 @@ function storedTreeBaseline(snapshot: SnapshotState | undefined): SurfaceScopedN */ async function observeSettled( context: GenericSettleContext, - settle: SettleParams, + settle: SettleParams & { baselineFresh: boolean }, baseline: SurfaceScopedNodes, ): Promise { const runtime = await createGenericSettleRuntime(context); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts new file mode 100644 index 0000000000..4b50808476 --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { + activateCompleteRefFrame, + expireRefFrame, + markSessionSnapshotOutdated, +} from '../../../core/ref-frame.ts'; +import { readSettleRequest } from './interaction-flags.ts'; + +const OPT_IN = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }; + +/** A session whose stored tree was published as a complete ref frame (`snapshot`). */ +function sessionWithFrame() { + const session = makeSession('default', { + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Next' }]), + }); + activateCompleteRefFrame(session); + return session; +} + +test('the host environment opts a settle request into the adaptive confirmation', () => { + const session = sessionWithFrame(); + + assert.deepEqual( + readSettleRequest( + { settle: true, settleQuietMs: 700, timeoutMs: 4_000 }, + { env: OPT_IN, session }, + ), + { quietMs: 700, timeoutMs: 4_000, earlyExit: 'adaptive', observationCurrent: true }, + ); +}); + +test('a ref frame a side effect crossed is not current for the adaptive confirmation', () => { + const session = sessionWithFrame(); + expireRefFrame(session); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a frame whose tree a native read outdated is not current for the adaptive confirmation', () => { + // `wait text` reads the device natively and marks the stored tree outdated without expiring the + // frame. + const session = sessionWithFrame(); + markSessionSnapshotOutdated(session); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a newer observation than the frame is not current for the adaptive confirmation', () => { + // An Android freshness capture stores a newer tree while the frame stays active. + const session = sessionWithFrame(); + session.snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Done' }]); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a settle request without the opt-in keeps the quiet window rule', () => { + const session = makeSession('default'); + const env = { AGENT_DEVICE_SETTLE_ADAPTIVE: 'true' }; + + assert.deepEqual(readSettleRequest({ settle: true }, { env, session }), {}); + assert.deepEqual(readSettleRequest({ settle: true }), {}); +}); + +test('the opt-in alone requests no settle', () => { + assert.equal(readSettleRequest({}, { env: OPT_IN, session: makeSession('default') }), undefined); +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts index 45abb96482..b0ea3db2fa 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts @@ -1,5 +1,8 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { CommandFlags } from '@agent-device/contracts/command'; +import type { EnvMap } from '@agent-device/kernel/source-value'; +import { isOutdatedObservation, refFrameState, refFrameTree } from '../../../core/ref-frame.ts'; +import type { SessionState } from '../../../core/session-state.ts'; import type { SettleParams } from '@agent-device/contracts/interaction'; import { interactionErrorResponse } from './interaction-response.ts'; @@ -17,10 +20,43 @@ export function settleFlagGuardResponse( ); } -export function readSettleRequest(flags: CommandFlags | undefined): SettleParams | undefined { +const ADAPTIVE_SETTLE_ENV = 'AGENT_DEVICE_SETTLE_ADAPTIVE'; + +/** + * The session's observation still describes the screen: the ref frame is active (no side-effect + * seam expired it), no native read marked its tree outdated (`wait text`), and no newer + * observation replaced it (an Android freshness capture). + */ +function isObservationCurrent(session: SessionState): boolean { + const tree = refFrameTree(session); + return ( + refFrameState(session) === 'active' && + tree !== undefined && + session.snapshot === tree && + !isOutdatedObservation(tree) + ); +} + +/** + * The settle request a command carries. `host` selects the internal adaptive confirmation prototype + * through the daemon host environment and nothing else. Read it before the action dispatches: the + * dispatch expires the ref frame, and the prototype asks whether the observation was current before + * it. + */ +export function readSettleRequest( + flags: CommandFlags | undefined, + host?: { env: Readonly; session: SessionState | undefined }, +): SettleParams | undefined { if (flags?.settle !== true) return undefined; + const adaptive = host?.env[ADAPTIVE_SETTLE_ENV] === '1'; return { ...(flags.settleQuietMs !== undefined ? { quietMs: flags.settleQuietMs } : {}), ...(flags.timeoutMs !== undefined ? { timeoutMs: flags.timeoutMs } : {}), + ...(adaptive + ? { + earlyExit: 'adaptive' as const, + observationCurrent: host.session !== undefined && isObservationCurrent(host.session), + } + : {}), }; } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts index 0cd73501b6..bc1efd50c3 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts @@ -12,7 +12,10 @@ import { refFrameState, refFrameTree, } from '../../../core/ref-frame.ts'; -import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + makeSessionStore, + makeSessionStoreWithEnv, +} from '../../../__tests__/support/store-factory.ts'; import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { clearRequestAbortRegistration, @@ -682,3 +685,51 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn expect(settle.settled).toBe(true); expect(typeof settle.refsGeneration).toBe('number'); }); + +// hover is web-only (refused on this iOS session); it reads the same shared settle options. +for (const command of ['press', 'click', 'longpress'] as const) { + for (const adaptive of [true, false]) { + test(`${command} --settle forwards ${adaptive ? 'the adaptive early exit' : 'only its window'} to the settle observation`, async () => { + const sessionStore = adaptive + ? makeSessionStoreWithEnv({ AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }) + : makeSessionStore(); + const sessionName = `settle-forward-${command}-${adaptive}`; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES] }); + const forwarded: unknown[] = []; + const recordingSurface = createTouchCommandSurface({ + observeSettle: async ({ settle }) => { + forwarded.push(settle); + return { settled: true, waitedMs: 0, captures: 1, quietMs: 25, timeoutMs: 2_000 }; + }, + }); + + const response = await handleInteractionCommands({ + createCommandSurface: recordingSurface, + req: { + token: 't', + session: sessionName, + command, + positionals: ['label=Continue'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + captureSnapshotForSession: mockCaptureSnapshotForSession, + ...getRuntimeBindings(), + }); + + expectOkData(response); + // The seeded snapshot published a complete frame and nothing outdated it, so the observation + // is current before the action. + expect(forwarded).toEqual([ + { + quietMs: 25, + timeoutMs: 2_000, + ...(adaptive ? { earlyExit: 'adaptive', observationCurrent: true } : {}), + }, + ]); + }); + } +} diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts index a5e114df65..d8e300d9a2 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts @@ -78,7 +78,10 @@ export async function dispatchFillViaRuntime(params: FillParams): Promise; flags: CommandFlags | undefined; + settle: ReturnType; durationMs?: number; expectedResolvedTarget?: ReplayTargetGuardDenotation; /** #1654: a mutating `find`'s already-resolved node; see daemon/daemon-request.ts. */ preresolvedTarget?: PreresolvedInteractionTarget; }): Promise { - const { runtime, command, target, sessionName, requestId, flags, expectedResolvedTarget } = - params; + const { runtime, command, target, sessionName, requestId, expectedResolvedTarget } = params; const shared = { session: sessionName, requestId, - settle: readSettleRequest(flags), + settle: params.settle, expectedResolvedTarget, }; switch (command) { diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index e8dc06b0c0..d36aded223 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -573,6 +573,20 @@ export function formatRole(type: string): string { return lookupRoleLabel(normalized) || normalized || 'element'; } +const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ + 'activity-indicator', + 'progress-indicator', + 'progressbar', +]); + +/** + * Whether a node is a busy indicator: a spinner or progress element that says only that the screen + * is not ready yet. Apple reports activity and progress indicators; Android reports a ProgressBar. + */ +export function isBusyIndicatorNode(node: Pick): boolean { + return BUSY_INDICATOR_ROLES.has(formatRole(node.type ?? '')); +} + /** * Reasons the Apple runner can stamp when serving a command required re-activating the session app * (#2682). Mirrors its `activateTarget(bundleId:reason:)` call sites.