From db668f164421c09181de4d8209bed91dcd4ec7ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:02:29 +0100 Subject: [PATCH 1/8] test(test-app): add a settle lab of timed UI effects Deep link agent-device-test-app://settle-lab. Each button starts one effect with a known final screen: an in-place counter, 500/800 ms delayed results, a fading overlay, two-stage transitions with and without a spinner, and a toast that precedes navigation. Co-Authored-By: Claude Opus 5.5 --- examples/test-app/README.md | 1 + examples/test-app/app/_layout.tsx | 2 + examples/test-app/app/settle-lab-result.tsx | 8 + examples/test-app/app/settle-lab.tsx | 5 + .../test-app/src/screens/SettleLabScreen.tsx | 183 ++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 examples/test-app/app/settle-lab-result.tsx create mode 100644 examples/test-app/app/settle-lab.tsx create mode 100644 examples/test-app/src/screens/SettleLabScreen.tsx diff --git a/examples/test-app/README.md b/examples/test-app/README.md index 96b05b33a9..81d287b764 100644 --- a/examples/test-app/README.md +++ b/examples/test-app/README.md @@ -19,6 +19,7 @@ It is intentionally small, but each surface is dense with durable accessibility - `Automation lab`: long-press, alert-result, app-event, app-state, appearance, orientation, permission-recovery, log canaries, a flattened (`accessible={true}`) text input, and an Apple Pay sheet hosted in `com.apple.PassbookUIService` - `WebView accessibility`: a deterministic semantic fixture plus live websites with varied HTML for native accessibility snapshot verification - `Perf lab`: seeded JavaScript render problems for `react-devtools profile`, one per scenario (see [Perf lab](#perf-lab)) +- `Settle lab`: timed effects for `--settle` measurements (in-place counter, 500 and 800 ms delayed results, a 750 ms fading overlay, two-stage transitions with and without a spinner, and a toast that precedes navigation), reached by the `agent-device-test-app://settle-lab` deep link - `Heavy feed`: a Bluesky-shaped home feed whose accessibility tree is deeper than 65 levels (see [Heavy feed](#heavy-feed)) Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface. diff --git a/examples/test-app/app/_layout.tsx b/examples/test-app/app/_layout.tsx index e452a411d2..38ca0c0e9f 100644 --- a/examples/test-app/app/_layout.tsx +++ b/examples/test-app/app/_layout.tsx @@ -31,6 +31,8 @@ function RootLayoutContent() { + + diff --git a/examples/test-app/app/settle-lab-result.tsx b/examples/test-app/app/settle-lab-result.tsx new file mode 100644 index 0000000000..d89fc10137 --- /dev/null +++ b/examples/test-app/app/settle-lab-result.tsx @@ -0,0 +1,8 @@ +import { useLocalSearchParams } from 'expo-router'; + +import { SettleLabResultScreen } from '../src/screens/SettleLabScreen'; + +export default function SettleLabResultRoute() { + const { run } = useLocalSearchParams<{ run?: string }>(); + return ; +} diff --git a/examples/test-app/app/settle-lab.tsx b/examples/test-app/app/settle-lab.tsx new file mode 100644 index 0000000000..47a8fa8625 --- /dev/null +++ b/examples/test-app/app/settle-lab.tsx @@ -0,0 +1,5 @@ +import { SettleLabScreen } from '../src/screens/SettleLabScreen'; + +export default function SettleLabRoute() { + return ; +} diff --git a/examples/test-app/src/screens/SettleLabScreen.tsx b/examples/test-app/src/screens/SettleLabScreen.tsx new file mode 100644 index 0000000000..3b2bd1fd9d --- /dev/null +++ b/examples/test-app/src/screens/SettleLabScreen.tsx @@ -0,0 +1,183 @@ +import { router } from 'expo-router'; +import { useEffect, useRef, useState } from 'react'; +import { ActivityIndicator, Animated, Pressable, StyleSheet, Text, View } from 'react-native'; + +/** + * Timed UI effects for measuring `--settle`: each button starts one effect whose final screen is + * known, so a settled tree can be checked against it. `settle-status` names the final state of the + * last effect with its run number. + */ +const DELAYS_MS = [500, 800] as const; +const FADE_MS = 750; +const STAGE_MS = 600; +const NAVIGATE_AFTER_TOAST_MS = 700; + +export function SettleLabScreen() { + const [run, setRun] = useState(0); + const [count, setCount] = useState(0); + const [status, setStatus] = useState('idle'); + const [stage, setStage] = useState<'none' | 'spinner' | 'plain'>('none'); + const [overlayVisible, setOverlayVisible] = useState(false); + const [toast, setToast] = useState(null); + const opacity = useRef(new Animated.Value(1)).current; + const timers = useRef[]>([]); + + useEffect(() => () => timers.current.forEach(clearTimeout), []); + + const later = (ms: number, effect: () => void) => { + timers.current.push(setTimeout(effect, ms)); + }; + const nextRun = () => { + const next = run + 1; + setRun(next); + return next; + }; + + return ( + + Settle lab + + {status} + + + {`Count ${count}`} + + setCount((c) => c + 1)} /> + {DELAYS_MS.map((delayMs) => ( + { + const id = nextRun(); + later(delayMs, () => setStatus(`Delayed ${delayMs} done ${id}`)); + }} + /> + ))} + { + const id = nextRun(); + opacity.setValue(1); + setOverlayVisible(true); + Animated.timing(opacity, { + toValue: 0, + duration: FADE_MS, + useNativeDriver: true, + }).start(() => { + setOverlayVisible(false); + setStatus(`Overlay gone ${id}`); + }); + }} + /> + { + const id = nextRun(); + setStage('spinner'); + later(STAGE_MS, () => { + setStage('none'); + setStatus(`Stage done ${id}`); + }); + }} + /> + { + const id = nextRun(); + setStage('plain'); + later(STAGE_MS, () => { + setStage('none'); + setStatus(`Stage done ${id}`); + }); + }} + /> + { + const id = nextRun(); + setToast('Saving'); + later(NAVIGATE_AFTER_TOAST_MS, () => { + setToast(null); + router.push({ pathname: '/settle-lab-result', params: { run: String(id) } }); + }); + }} + /> + {stage === 'spinner' ? ( + + + Working + + ) : null} + {stage === 'plain' ? ( + + Step 1 of 2 + + ) : null} + {toast ? ( + + {toast} + + ) : null} + {overlayVisible ? ( + + + Overlay + + + ) : null} + + ); +} + +export function SettleLabResultScreen({ run }: { run: string | undefined }) { + return ( + + {`Result ${run ?? ''}`} + router.back()} /> + + ); +} + +function LabButton(props: { id: string; label: string; onPress: () => void }) { + return ( + + {props.label} + + ); +} + +const styles = StyleSheet.create({ + frame: { flex: 1, gap: 8, padding: 24, paddingTop: 72, backgroundColor: '#fff' }, + title: { fontSize: 24, fontWeight: '700' }, + status: { fontSize: 16 }, + row: { flexDirection: 'row', gap: 8, alignItems: 'center' }, + button: { backgroundColor: '#e8eefc', borderRadius: 8, padding: 12 }, + buttonLabel: { fontSize: 16 }, + toast: { + position: 'absolute', + bottom: 48, + left: 24, + right: 24, + backgroundColor: '#222', + borderRadius: 8, + padding: 12, + }, + toastLabel: { color: '#fff', fontSize: 16 }, + overlay: { + ...StyleSheet.absoluteFill, + alignItems: 'center', + justifyContent: 'center', + backgroundColor: 'rgba(0,0,0,0.5)', + }, + overlayButton: { backgroundColor: '#333', borderRadius: 8, padding: 24 }, +}); From 421179e5d6fcc892555b1033da53c49d2f60fe52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:02:29 +0100 Subject: [PATCH 2/8] feat(settle): prototype an adaptive early exit behind an internal switch A capture that differs from a fresh pre-action baseline settles once one confirming capture ~110 ms later agrees with it. A disagreeing pair, a busy indicator, a producer or presented-surface change, the broad-replacement floor, a capture that holds both screens of a transition, or a baseline the action did not capture while the daemon reports the observation not current keep today's quiet window. The daemon enables it only when its host environment sets AGENT_DEVICE_SETTLE_ADAPTIVE=1; default off, no flag or output change. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/settle-device-fixtures.ts | 3 + .../settle-adaptive-confirmation.test.ts | 262 ++++++++++++++++++ .../runtime/settle-adaptive-confirmation.ts | 140 ++++++++++ .../commands/interaction/runtime/settle.ts | 30 +- .../runtime/stable-capture-signal.ts | 7 +- .../interaction/runtime/stable-capture.ts | 30 +- .../contracts/src/daemon-command-surface.ts | 2 + packages/contracts/src/interaction.ts | 12 + .../src/core/deferred-interaction-outcome.ts | 9 +- .../daemon/src/execution/generic-settle.ts | 20 +- .../internal/interaction-flags.test.ts | 75 +++++ .../interaction/internal/interaction-flags.ts | 38 ++- .../internal/interaction-touch-fill.ts | 5 +- .../internal/interaction-touch-press.ts | 7 +- packages/kernel/src/snapshot.ts | 14 + 15 files changed, 628 insertions(+), 26 deletions(-) create mode 100644 packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts create mode 100644 packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts create mode 100644 packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts diff --git a/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts b/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts index ea6c7f0962..4d610652b3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/__tests__/settle-device-fixtures.ts @@ -60,6 +60,8 @@ export function welcomeSnapshot(): SnapshotState { export function createSettleDevice(params: { stored: SnapshotState; + /** The ADR 0014 ref frame, when the scenario issued one apart from the stored observation. */ + refFrame?: SnapshotState; captureSnapshot: () => Promise | BackendSnapshotResult; tap?: () => Promise>; clock?: ReturnType; @@ -78,6 +80,7 @@ export function createSettleDevice(params: { { name: 'default', snapshot: params.stored, + ...(params.refFrame ? { refFrameSnapshot: params.refFrame } : {}), ...(params.appBundleId ? { appBundleId: params.appBundleId } : {}), }, ]), diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts new file mode 100644 index 0000000000..5460b8e8ba --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts @@ -0,0 +1,262 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { SnapshotProvenance, SnapshotState } from '@agent-device/contracts/snapshot-model'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { ref, selector } from './selector-read-utils.ts'; +import { createSettleDevice, welcomeSnapshot } from './__tests__/settle-device-fixtures.ts'; +import { + elementCatalogTabSnapshot, + elementSettingsTabSnapshot, +} from './stable-capture.fixtures.ts'; + +// The internal adaptive confirmation prototype: a capture that differs from a fresh pre-action +// baseline settles once one short confirmation agrees with it; every escalation keeps the quiet +// window. Each case runs the same capture sequence with the switch on and off, so the difference +// is the rule's and not the fixture's. Budgets run on the injected clock; captures cost nothing. + +const DEFAULT_QUIET_MS = 500; + +/** The welcome screen after a press relabels its heading in place: no broad replacement. */ +function relabeled(label = 'Welcome back!'): SnapshotState { + const base = welcomeSnapshot(); + return { + ...base, + nodes: base.nodes.map((node) => (node.index === 0 ? { ...node, label } : node)), + }; +} + +function withProvenance

(state: SnapshotState, provenance: P) { + return { ...state, ...provenance }; +} + +function withSpinner(snapshot: SnapshotState): SnapshotState { + return makeSnapshotState([ + ...snapshot.nodes, + { + index: snapshot.nodes.length, + depth: 1, + parentIndex: 0, + type: 'ActivityIndicator', + label: 'In progress', + rect: { x: 10, y: 400, width: 20, height: 20 }, + }, + ]); +} + +/** + * An @ref press on the stored tree, whose loop captures follow `loop` (the last one repeats). The + * native ref route captures nothing before the action, so the stored tree is the baseline, fresh + * here because the daemon reports that observation current. + */ +async function press(params: { + stored?: SnapshotState; + loop: SnapshotState[]; + adaptive: boolean; + target?: Parameters['interactions']['press']>[0]; +}) { + let captures = 0; + const device = createSettleDevice({ + stored: params.stored ?? welcomeSnapshot(), + captureSnapshot: () => { + captures += 1; + return { snapshot: params.loop[Math.min(captures, params.loop.length) - 1]! }; + }, + }); + const result = await device.interactions.press(params.target ?? ref('@e2'), { + session: 'default', + settle: params.adaptive ? { earlyExit: 'adaptive', observationCurrent: true } : {}, + }); + assert.ok(result.settle); + return result.settle; +} + +test('a changed capture that the next capture confirms settles before the quiet window', async () => { + const loop = [relabeled(), relabeled()]; + + const adaptive = await press({ loop, adaptive: true }); + const quiet = await press({ loop, adaptive: false }); + + assert.equal(adaptive.settled, true); + assert.equal(adaptive.captures, 2); + assert.ok(adaptive.waitedMs < 200, `adaptive held ${adaptive.waitedMs}ms`); + assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!'))); + assert.equal(quiet.settled, true); + assert.ok(quiet.waitedMs >= DEFAULT_QUIET_MS, `quiet window held ${quiet.waitedMs}ms`); +}); + +test('a confirmation that disagrees falls back to the full quiet window', async () => { + const adaptive = await press({ + loop: [relabeled('Loading'), relabeled('Welcome back!')], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); + assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!'))); +}); + +test('a capture that still matches the baseline keeps the quiet window', async () => { + const adaptive = await press({ loop: [welcomeSnapshot()], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a change that arrives after an unchanged capture is confirmed on its own', async () => { + const adaptive = await press({ + loop: [welcomeSnapshot(), relabeled(), relabeled()], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.equal(adaptive.captures, 3); + assert.ok(adaptive.waitedMs < DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a busy indicator on the changed screen keeps the quiet window', async () => { + const loop = [withSpinner(relabeled())]; + + const adaptive = await press({ loop, adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a confirmation from another producer keeps the quiet window', async () => { + const adaptive = await press({ + loop: [ + withProvenance(relabeled(), { backend: 'xctest', producer: 'apple-runner' }), + withProvenance(relabeled(), { backend: 'xctest', producer: 'appium-source' }), + ], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a presented surface the baseline did not show keeps the quiet window', async () => { + const adaptive = await press({ + stored: { ...welcomeSnapshot(), presentedSurface: 'absent' }, + loop: [{ ...relabeled(), presentedSurface: 'presented' }], + adaptive: true, + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +test('a broad replacement owed the confirmation floor keeps the floor', async () => { + const settings = elementSettingsTabSnapshot(); + + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [settings], + adaptive: true, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= 1_500, `adaptive lost the floor: ${adaptive.waitedMs}ms`); +}); + +test('a point press whose baseline is the stored tree keeps the quiet window', async () => { + // The pre-action evidence capture fails, so the baseline falls back to the stored tree, which may + // predate a side effect: a change against it does not prove the action's effect. + let captures = 0; + const device = createSettleDevice({ + stored: welcomeSnapshot(), + captureSnapshot: () => { + captures += 1; + if (captures === 1) throw new Error('capture unavailable'); + return { snapshot: relabeled() }; + }, + }); + + const result = await device.interactions.press( + { kind: 'point', x: 50, y: 100 }, + { session: 'default', settle: { earlyExit: 'adaptive' } }, + ); + + assert.equal(result.settle?.settled, true); + assert.ok(result.settle.waitedMs >= DEFAULT_QUIET_MS, `held ${result.settle.waitedMs}ms`); +}); + +test('a capture that holds the baseline beside a new screen keeps the quiet window', async () => { + // A navigation transition can publish both screens at once and hold that union across captures. + const base = welcomeSnapshot(); + const union = makeSnapshotState([ + ...base.nodes, + ...['Detail', 'Buy', 'Share'].map((label, offset) => ({ + index: base.nodes.length + offset, + depth: 1, + parentIndex: 0, + type: 'Button', + label, + rect: { x: 200, y: 20 + offset * 60, width: 100, height: 40 }, + })), + ]); + + const adaptive = await press({ loop: [union], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); + +for (const [label, observationCurrent, early] of [ + ['a current observation is a fresh baseline', true, true], + ['a frame the daemon reports outdated keeps the quiet window', false, false], +] as const) { + test(`a selector press read in another view than the frame: ${label}`, async () => { + // The resolution capture carries no presentation, the frame is interactive-only, so the settle + // baseline is the frame itself and only the daemon can say whether it is current. + const frame = { ...welcomeSnapshot(), presentation: { interactiveOnly: true } }; + let captures = 0; + const device = createSettleDevice({ + stored: frame, + refFrame: frame, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? welcomeSnapshot() : relabeled() }; + }, + }); + + const result = await device.interactions.press(selector('label=Next'), { + session: 'default', + settle: { earlyExit: 'adaptive', observationCurrent }, + }); + + assert.equal(result.settle?.settled, true); + assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`); + }); +} + +for (const [label, observationCurrent, early] of [ + ['a current observation is a fresh baseline', true, true], + ['a frame the daemon reports outdated keeps the quiet window', false, false], +] as const) { + test(`an @ref press on the stored frame: ${label}`, async () => { + // The ref route resolves against the stored frame and captures nothing before the action, so the + // frame is the baseline. After `wait text` (or an Android freshness read) the frame stays active + // but no longer describes the screen, and only the daemon knows that. + const frame = welcomeSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: frame, + refFrame: frame, + captureSnapshot: () => { + captures += 1; + return { snapshot: relabeled() }; + }, + }); + + const result = await device.interactions.press(ref('@e2'), { + session: 'default', + settle: { earlyExit: 'adaptive', observationCurrent }, + }); + + assert.equal(captures > 0, true); + assert.equal(result.settle?.settled, true); + assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`); + }); +} diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts new file mode 100644 index 0000000000..b157e36f85 --- /dev/null +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -0,0 +1,140 @@ +import { isBusyIndicatorNode } from '@agent-device/kernel/snapshot'; +import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction'; +import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; +import { + sharedIdentityCount, + stableCaptureSignal, + stableCaptureSignalsEqual, + type StableCaptureSignal, +} from './stable-capture-signal.ts'; + +/** + * Internal prototype (`SettleParams.earlyExit: 'adaptive'`): a capture that differs from a fresh + * pre-action baseline, and that one short confirmation capture agrees with, settles before the + * quiet window. Every other case keeps the quiet window: no capture has differed yet (a no-op, a + * toggle that has not rendered, a late effect), the confirmation disagrees, a busy indicator is on + * screen, the producer or the presented surface changed, the broad-replacement floor is in force, + * the capture still holds the baseline screen beside a new one, or the baseline may predate a side + * effect. One failed check disarms the rule for the rest of the run. + * + * Two equal captures ~110 ms apart cannot exclude a later stage nothing marks: a toast standing in + * for navigation, a plateau between transition stages, a late value write. That is the trade this + * prototype measures. + */ +export const ADAPTIVE_CONFIRMATION_GAP_MS = 110; + +export function createAdaptiveConfirmation(params: { + baseline: SurfaceScopedNodes | undefined; + baselineFresh: boolean; + appBundleId: string | undefined; +}) { + const { baseline, appBundleId } = params; + let armed = params.baselineFresh && baseline !== undefined && baseline.nodes.length > 0; + let candidate: { signal: StableCaptureSignal; snapshot: SnapshotState } | undefined; + const disarm = () => { + armed = false; + candidate = undefined; + }; + return { + /** + * Folds one capture in. `settle`: this capture confirmed the previous one. `confirm`: take the + * next capture after the short gap. `none`: the quiet window decides. + */ + observe(input: { + snapshot: SnapshotState; + signal: StableCaptureSignal; + /** The broad-replacement floor raised this capture's window above the producer's own. */ + floorInForce: boolean; + }): 'settle' | 'confirm' | 'none' { + if (!armed || !baseline) return 'none'; + const { snapshot, signal } = input; + if ( + input.floorInForce || + hasBusyIndicator(snapshot) || + isTransitionalUnion(baseline, snapshot, appBundleId) || + presentedSurfaceChanged(baseline, snapshot) + ) { + disarm(); + return 'none'; + } + if (candidate) { + const confirmed = + sameProvenance(candidate.snapshot, snapshot) && + stableCaptureSignalsEqual(candidate.signal, signal); + disarm(); + return confirmed ? 'settle' : 'none'; + } + if (!differsFromBaseline(baseline, snapshot, appBundleId)) return 'none'; + candidate = { signal, snapshot }; + return 'confirm'; + }, + /** A private-AX recovery re-read the screen through another backend. */ + disarm, + }; +} + +/** + * A navigation transition can publish both screens at once, and that union can hold still across + * two captures. A capture that keeps most of the baseline while adding half as much again is that + * union, not a settled screen. + */ +const UNION_RETAINED_RATIO = 0.8; +const UNION_GROWTH_RATIO = 0.5; + +function isTransitionalUnion( + baseline: SurfaceScopedNodes, + snapshot: SnapshotState, + appBundleId: string | undefined, +): boolean { + const before = projectedSignal(snapshot, baseline.nodes, appBundleId); + const after = projectedSignal(snapshot, snapshot.nodes, appBundleId); + if (before.nodes.length === 0) return false; + const shared = sharedIdentityCount(before, after); + return ( + shared / before.nodes.length >= UNION_RETAINED_RATIO && + (after.nodes.length - shared) / before.nodes.length >= UNION_GROWTH_RATIO + ); +} + +/** + * One side of a baseline comparison: both sides take the capture's provenance so visibility + * filtering applies alike, and both drop the keyboard chrome the settle diff collapses. + */ +function projectedSignal( + snapshot: SnapshotState, + nodes: SnapshotState['nodes'], + appBundleId: string | undefined, +): StableCaptureSignal { + return stableCaptureSignal({ ...snapshot, nodes: withoutSettleChrome(nodes, appBundleId) }); +} + +function differsFromBaseline( + baseline: SurfaceScopedNodes, + snapshot: SnapshotState, + appBundleId: string | undefined, +): boolean { + // A keyboard alone is not the action's effect: the projection drops it. + return !stableCaptureSignalsEqual( + projectedSignal(snapshot, baseline.nodes, appBundleId), + projectedSignal(snapshot, snapshot.nodes, appBundleId), + ); +} + +function hasBusyIndicator(snapshot: SnapshotState): boolean { + return snapshot.nodes.some(isBusyIndicatorNode); +} + +function presentedSurfaceChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean { + const before = baseline.presentedSurface; + const after = snapshot.presentedSurface; + return before !== undefined && after !== undefined && before !== after; +} + +function sameProvenance(left: SnapshotState, right: SnapshotState): boolean { + return ( + left.producer === right.producer && + left.backend === right.backend && + left.presentedSurface === right.presentedSurface + ); +} diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts index dd0c64a38d..8aee0cb71c 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts @@ -92,7 +92,7 @@ export async function settleAfterInteraction( ): Promise { return await settleAfterAction(runtime, options, { ...params, - baseline: await resolveSettleBaseline(runtime, options, params.resolved), + ...(await resolveSettleBaseline(runtime, options, params.resolved, params.observationCurrent)), actionPoint: params.resolved.point, }); } @@ -101,6 +101,11 @@ export type SettleObservationCommandOptions = CommandContext & SettleParams & { /** The pre-action tree the settled diff is taken against, and the surface it describes. */ baseline: SurfaceScopedNodes; + /** + * Whether `baseline` describes the screen right before this action. Absent means it may predate + * a side effect, so the adaptive confirmation never treats a change against it as the action's. + */ + baselineFresh?: boolean; }; /** @@ -133,7 +138,11 @@ export const settleObservationCommand: RuntimeCommand< async function settleAfterAction( runtime: AgentDeviceRuntime, options: CommandContext, - params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point }, + params: SettleParams & { + baseline: SurfaceScopedNodes; + baselineFresh?: boolean; + actionPoint?: Point; + }, ): Promise { // The loop resolves an unrequested window from the capture that pays it, so the window is only // known once one is taken. `base` carries the general default for the pre-capture shape the @@ -154,6 +163,9 @@ async function settleAfterAction( timeoutMs, resetBudgetOnPrivateAxRecovery: true, broadTransitionBaseline: params.baseline, + ...(params.earlyExit === 'adaptive' + ? { adaptiveConfirmation: { baselineFresh: params.baselineFresh === true } } + : {}), }); if (outcome.failure) return settleUnavailable(base, outcome, outcome.failure.error); return await readSettledOutcome(runtime, options, params, base, outcome); @@ -237,9 +249,11 @@ async function resolveSettleBaseline( runtime: AgentDeviceRuntime, options: CommandContext, resolved: ResolvedInteractionTarget, -): Promise { + observationCurrent: boolean | undefined, +): Promise<{ baseline: SurfaceScopedNodes; baselineFresh: boolean }> { const session = await runtime.sessions.get(options.session ?? 'default'); const frame = surfaceScopedNodes(session?.refFrameSnapshot); + const refFrame = resolved.kind === 'ref' ? frame : undefined; // A ref is authorized against the stored ref frame. Keep that visible presentation as the // transition baseline: a best-effort evidence recapture can recover through private AX and see // covered background controls that were not actionable when the ref was issued. @@ -247,16 +261,22 @@ async function resolveSettleBaseline( // authoritative ref frame, so reuse it rather than silently turning a missing optional field // into an empty transition/diff baseline. Fall back to the latest observation for point targets // and pre-frame sessions. A candidate read in another view than the frame is skipped (#4486). - return postActionBaseline({ + const baseline = await postActionBaseline({ frame: withoutPresentedSurface(frame), candidates: [ - resolved.kind === 'ref' ? frame : undefined, + refFrame, resolved.preAction, session?.refFrameSnapshot ? undefined : withoutPresentedSurface(surfaceScopedNodes(session?.snapshot)), ], }); + // Only a capture this action took (a selector resolution or a point's evidence read) proves the + // screen before it. A ref resolves against the stored frame, so its `preAction` is that frame too; + // the frame and the stored tree are fresh only while the daemon reports the observation current. + const actionCapture = resolved.kind !== 'ref' && baseline === resolved.preAction; + const baselineFresh = baseline.nodes.length > 0 && (actionCapture || observationCurrent === true); + return { baseline, baselineFresh }; } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts index fcc6a02c47..e2a37cd415 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts @@ -53,6 +53,11 @@ export function stableCaptureSignalsHaveBroadReplacement( // semantic replacement. Both signals carry the same chrome-free visible projection the settle // diff collapses, and the transition baseline is only armed for XCTest sessions. if (!left || left.nodes.length === 0 || right.nodes.length === 0) return false; + return sharedIdentityCount(left, right) / Math.max(left.nodes.length, right.nodes.length) < 0.5; +} + +/** How many nodes the two signals share by semantic identity, each node matched at most once. */ +export function sharedIdentityCount(left: StableCaptureSignal, right: StableCaptureSignal): number { const remainingIdentities = new Map(); for (const node of left.nodes) { remainingIdentities.set(node.identity, (remainingIdentities.get(node.identity) ?? 0) + 1); @@ -64,7 +69,7 @@ export function stableCaptureSignalsHaveBroadReplacement( sharedNodes += 1; remainingIdentities.set(node.identity, remaining - 1); } - return sharedNodes / Math.max(left.nodes.length, right.nodes.length) < 0.5; + return sharedNodes; } /** diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 0060264cd2..dd39dcc56a 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -25,6 +25,10 @@ import { } from './stable-capture-signal.ts'; import { preferredSnapshotBackendForVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; +import { + ADAPTIVE_CONFIRMATION_GAP_MS, + createAdaptiveConfirmation, +} from './settle-adaptive-confirmation.ts'; /** * The quiet-window stable-capture loop shared by `wait stable` and the @@ -110,6 +114,11 @@ export async function runStableCaptureLoop( * captures in its presentation, so the settled tree and the baseline share one view (#4486). */ broadTransitionBaseline?: SurfaceScopedNodes; + /** + * Internal prototype: settle on a baseline-differing capture that one short confirmation agrees + * with (see settle-adaptive-confirmation.ts). Absent keeps the quiet window alone. + */ + adaptiveConfirmation?: { baselineFresh: boolean }; }, ): Promise { const { timeoutMs } = params; @@ -135,6 +144,13 @@ export async function runStableCaptureLoop( // loop recovers on private AX, the rest of the loop stays on it. let preferredBackend: SnapshotPreferredBackend | undefined; const pacing = createStableCapturePacing(start); + const adaptive = params.adaptiveConfirmation + ? createAdaptiveConfirmation({ + baseline: params.broadTransitionBaseline, + baselineFresh: params.adaptiveConfirmation.baselineFresh, + appBundleId: session?.appBundleId, + }) + : undefined; let captures = 0; let lastCapture: CapturedSnapshot | undefined; const result = ( @@ -199,13 +215,25 @@ export async function runStableCaptureLoop( if (recovered) { privateAxRecoveryBudgetReset = true; deadlineMs = recovered.deadlineMs; + adaptive?.disarm(); } const step = recovered ? { settled: false, delayMs: recovered.delayMs } : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures }); if (step.settled) return result(true, false, nowMs - start); + const early = recovered + ? 'none' + : adaptive?.observe({ + snapshot: capture.snapshot, + signal, + floorInForce: requiredQuietMs > window.quietMs, + }); + if (early === 'settle') return result(true, false, nowMs - start); if (step.delayMs <= 0) break; - await sleep(runtime, step.delayMs); + await sleep( + runtime, + early === 'confirm' ? Math.min(step.delayMs, ADAPTIVE_CONFIRMATION_GAP_MS) : step.delayMs, + ); } return result(false, false, now(runtime) - start); } diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 85e86560bc..03c4c1cfe2 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -300,6 +300,8 @@ export type DaemonSettleObservationOptions = CommandContext & Readonly<{ /** The pre-action tree the settled diff is taken against, and the surface it describes. */ baseline: SurfaceScopedNodes; + /** Whether `baseline` describes the screen right before this action (adaptive settle only). */ + baselineFresh?: boolean; }>; /** diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index edd81f0787..8f0b5327bd 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -301,6 +301,18 @@ export type SettleTailEntry = { export type SettleParams = { quietMs?: number; timeoutMs?: number; + /** + * Internal prototype switch, never a CLI flag: 'adaptive' lets a baseline-differing capture that one + * short confirmation agrees with settle before the quiet window. The daemon sets it only when the + * host environment opts in (AGENT_DEVICE_SETTLE_ADAPTIVE=1). + */ + earlyExit?: 'adaptive'; + /** + * With `earlyExit`: when the action began, the session's ref frame was active, its tree was the + * newest observation, and no native read had outdated it. A baseline this action did not capture + * itself (the ref frame or the stored tree) is fresh only then. + */ + observationCurrent?: boolean; }; /** diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 1ef7baa1ca..807fc0dbb4 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -3,7 +3,7 @@ import type { SettleObservation, SurfaceScopedNodes } from '@agent-device/contra import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime'; import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device'; -import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot'; +import { formatRole, isBusyIndicatorNode, type SnapshotNode } from '@agent-device/kernel/snapshot'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { @@ -290,7 +290,7 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { const reading = node.label?.trim() || node.value?.trim(); if (!reading) continue; const role = formatRole(node.type ?? ''); - if (BUSY_INDICATOR_ROLES.has(role)) continue; + if (isBusyIndicatorNode(node)) continue; if (role !== 'text') return true; texts.add(reading); if (texts.size > 1) return true; @@ -298,11 +298,6 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean { return false; } -const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ - 'activity-indicator', - 'progress-indicator', -]); - function clearSurfaceStabilization(session: SessionState | undefined): void { if (session?.deferredSurface?.kind !== 'stabilize') return; session.deferredSurface = undefined; diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts index f2accd1537..3d9cfd889c 100644 --- a/packages/daemon/src/execution/generic-settle.ts +++ b/packages/daemon/src/execution/generic-settle.ts @@ -90,16 +90,22 @@ export async function planGenericSettleObservation( if (!commandSupportsSettleObservation(params.command)) return {}; const invalidSettleFlags = settleFlagGuardResponse(params.command, params.flags); if (invalidSettleFlags) return { response: invalidSettleFlags }; - const settle = readSettleRequest(params.flags); + const settle = readSettleRequest(params.flags, { + env: params.sessionStore.host.env, + session: params.session, + }); if (!settle) return {}; const { presentedSurface: _unverified, ...frame } = surfaceScopedNodes( refFrameTree(params.session), ); - const baseline = await postActionBaseline({ - frame, - candidates: [storedTreeBaseline(params.session.snapshot)], - }); - return { observe: async () => await observeSettled(params, settle, baseline) }; + const stored = storedTreeBaseline(params.session.snapshot); + const baseline = await postActionBaseline({ frame, candidates: [stored] }); + // The generic route captures nothing before the action, so the stored tree is fresh only while the + // daemon reports the session observation current. + const baselineFresh = baseline === stored && settle.observationCurrent === true; + return { + observe: async () => await observeSettled(params, { ...settle, baselineFresh }, baseline), + }; } /** @@ -124,7 +130,7 @@ function storedTreeBaseline(snapshot: SnapshotState | undefined): SurfaceScopedN */ async function observeSettled( context: GenericSettleContext, - settle: SettleParams, + settle: SettleParams & { baselineFresh: boolean }, baseline: SurfaceScopedNodes, ): Promise { const runtime = await createGenericSettleRuntime(context); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts new file mode 100644 index 0000000000..4bb7d0c2cb --- /dev/null +++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { makeSession } from '../../../__tests__/support/session-factories.ts'; +import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; +import { + activateCompleteRefFrame, + expireRefFrame, + markSessionSnapshotOutdated, +} from '../../../core/ref-frame.ts'; +import { readSettleRequest } from './interaction-flags.ts'; + +const OPT_IN = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }; + +/** A session whose stored tree was published as a complete ref frame (`snapshot`). */ +function sessionWithFrame() { + const session = makeSession('default', { + snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Next' }]), + }); + activateCompleteRefFrame(session); + return session; +} + +test('the host environment opts a settle request into the adaptive confirmation', () => { + const session = sessionWithFrame(); + + assert.deepEqual(readSettleRequest({ settle: true }, { env: OPT_IN, session }), { + earlyExit: 'adaptive', + observationCurrent: true, + }); +}); + +test('a ref frame a side effect crossed is not current for the adaptive confirmation', () => { + const session = sessionWithFrame(); + expireRefFrame(session); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a frame whose tree a native read outdated is not current for the adaptive confirmation', () => { + // `wait text` reads the device natively and marks the stored tree outdated without expiring the + // frame. + const session = sessionWithFrame(); + markSessionSnapshotOutdated(session); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a newer observation than the frame is not current for the adaptive confirmation', () => { + // An Android freshness capture stores a newer tree while the frame stays active. + const session = sessionWithFrame(); + session.snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Done' }]); + + assert.equal( + readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent, + false, + ); +}); + +test('a settle request without the opt-in keeps the quiet window rule', () => { + const session = makeSession('default'); + const env = { AGENT_DEVICE_SETTLE_ADAPTIVE: 'true' }; + + assert.deepEqual(readSettleRequest({ settle: true }, { env, session }), {}); + assert.deepEqual(readSettleRequest({ settle: true }), {}); +}); + +test('the opt-in alone requests no settle', () => { + assert.equal(readSettleRequest({}, { env: OPT_IN, session: makeSession('default') }), undefined); +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts index 45abb96482..b0ea3db2fa 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts @@ -1,5 +1,8 @@ import type { DaemonResponse } from '@agent-device/kernel/contracts'; import type { CommandFlags } from '@agent-device/contracts/command'; +import type { EnvMap } from '@agent-device/kernel/source-value'; +import { isOutdatedObservation, refFrameState, refFrameTree } from '../../../core/ref-frame.ts'; +import type { SessionState } from '../../../core/session-state.ts'; import type { SettleParams } from '@agent-device/contracts/interaction'; import { interactionErrorResponse } from './interaction-response.ts'; @@ -17,10 +20,43 @@ export function settleFlagGuardResponse( ); } -export function readSettleRequest(flags: CommandFlags | undefined): SettleParams | undefined { +const ADAPTIVE_SETTLE_ENV = 'AGENT_DEVICE_SETTLE_ADAPTIVE'; + +/** + * The session's observation still describes the screen: the ref frame is active (no side-effect + * seam expired it), no native read marked its tree outdated (`wait text`), and no newer + * observation replaced it (an Android freshness capture). + */ +function isObservationCurrent(session: SessionState): boolean { + const tree = refFrameTree(session); + return ( + refFrameState(session) === 'active' && + tree !== undefined && + session.snapshot === tree && + !isOutdatedObservation(tree) + ); +} + +/** + * The settle request a command carries. `host` selects the internal adaptive confirmation prototype + * through the daemon host environment and nothing else. Read it before the action dispatches: the + * dispatch expires the ref frame, and the prototype asks whether the observation was current before + * it. + */ +export function readSettleRequest( + flags: CommandFlags | undefined, + host?: { env: Readonly; session: SessionState | undefined }, +): SettleParams | undefined { if (flags?.settle !== true) return undefined; + const adaptive = host?.env[ADAPTIVE_SETTLE_ENV] === '1'; return { ...(flags.settleQuietMs !== undefined ? { quietMs: flags.settleQuietMs } : {}), ...(flags.timeoutMs !== undefined ? { timeoutMs: flags.timeoutMs } : {}), + ...(adaptive + ? { + earlyExit: 'adaptive' as const, + observationCurrent: host.session !== undefined && isObservationCurrent(host.session), + } + : {}), }; } diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts index a5e114df65..d8e300d9a2 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts @@ -78,7 +78,10 @@ export async function dispatchFillViaRuntime(params: FillParams): Promise; flags: CommandFlags | undefined; + settle: ReturnType; durationMs?: number; expectedResolvedTarget?: ReplayTargetGuardDenotation; /** #1654: a mutating `find`'s already-resolved node; see daemon/daemon-request.ts. */ preresolvedTarget?: PreresolvedInteractionTarget; }): Promise { - const { runtime, command, target, sessionName, requestId, flags, expectedResolvedTarget } = - params; + const { runtime, command, target, sessionName, requestId, expectedResolvedTarget } = params; const shared = { session: sessionName, requestId, - settle: readSettleRequest(flags), + settle: params.settle, expectedResolvedTarget, }; switch (command) { diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts index e8dc06b0c0..d36aded223 100644 --- a/packages/kernel/src/snapshot.ts +++ b/packages/kernel/src/snapshot.ts @@ -573,6 +573,20 @@ export function formatRole(type: string): string { return lookupRoleLabel(normalized) || normalized || 'element'; } +const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([ + 'activity-indicator', + 'progress-indicator', + 'progressbar', +]); + +/** + * Whether a node is a busy indicator: a spinner or progress element that says only that the screen + * is not ready yet. Apple reports activity and progress indicators; Android reports a ProgressBar. + */ +export function isBusyIndicatorNode(node: Pick): boolean { + return BUSY_INDICATOR_ROLES.has(formatRole(node.type ?? '')); +} + /** * Reasons the Apple runner can stamp when serving a command required re-activating the session app * (#2682). Mirrors its `activateTarget(bundleId:reason:)` call sites. From c423b993eb7e7a30700a7507cb1e8640b7c79b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:14:27 +0100 Subject: [PATCH 3/8] refactor(settle): keep the adaptive early exit off the eager closure and the loop under threshold Load the adaptive module only when a settle asks for it, fold one capture's quiet-window and early-exit decisions into observeStableCapture, and name the capture presentation. The wait command reuses the contracts DaemonWaitTarget instead of a copy. Co-Authored-By: Claude Opus 5.5 --- .../interaction/runtime/selector-wait.ts | 22 +--- .../runtime/settle-adaptive-confirmation.ts | 6 +- .../interaction/runtime/stable-capture.ts | 116 +++++++++++++----- .../contracts/src/daemon-command-surface.ts | 7 +- 4 files changed, 99 insertions(+), 52 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts b/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts index 2c7d4de739..ac0eb8dec3 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts @@ -1,6 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { findNodeByRef, normalizeRef, type SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { TargetAnnotationV1 } from '@agent-device/contracts/replay'; +import type { DaemonWaitTarget } from '@agent-device/contracts/daemon-command-surface'; import type { PublicPlatform } from '@agent-device/kernel/device'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { checkWaitText, STALE_REF_HINT } from '@agent-device/selectors'; @@ -27,25 +27,7 @@ export type SelectorSnapshotOptions = { export type WaitCommandOptions = WaitCommandContext & SelectorSnapshotOptions & { - target: - | { kind: 'sleep'; durationMs: number } - | { kind: 'text'; text: string; timeoutMs?: number | null } - | { kind: 'ref'; ref: string; timeoutMs?: number | null } - | { - kind: 'selector'; - selector: string; - timeoutMs?: number | null; - /** - * ADR 0012 / #1349, replay-only: the recorded landmark identity this - * wait must observe before reporting success. Polling is unchanged — - * the loop keeps waiting while no selector match carries this - * identity, and a timeout with rejected candidates throws the - * `WAIT_LANDMARK_MISMATCH_REASON` refusal instead of success. - */ - recordedLandmark?: TargetAnnotationV1; - } - | { kind: 'absent'; selector: string; timeoutMs?: number | null } - | { kind: 'stable'; quietMs?: number | null; timeoutMs?: number | null }; + target: DaemonWaitTarget; }; export type WaitCommandResult = diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts index b157e36f85..3086787b27 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -22,7 +22,9 @@ import { * for navigation, a plateau between transition stages, a late value write. That is the trade this * prototype measures. */ -export const ADAPTIVE_CONFIRMATION_GAP_MS = 110; +const ADAPTIVE_CONFIRMATION_GAP_MS = 110; + +export type AdaptiveEarlyExit = ReturnType; export function createAdaptiveConfirmation(params: { baseline: SurfaceScopedNodes | undefined; @@ -71,6 +73,8 @@ export function createAdaptiveConfirmation(params: { }, /** A private-AX recovery re-read the screen through another backend. */ disarm, + /** The wait before the confirming capture. */ + confirmGapMs: ADAPTIVE_CONFIRMATION_GAP_MS, }; } diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index dd39dcc56a..5dd494cb6b 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -25,10 +25,7 @@ import { } from './stable-capture-signal.ts'; import { preferredSnapshotBackendForVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict'; import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome'; -import { - ADAPTIVE_CONFIRMATION_GAP_MS, - createAdaptiveConfirmation, -} from './settle-adaptive-confirmation.ts'; +import type { AdaptiveEarlyExit } from './settle-adaptive-confirmation.ts'; /** * The quiet-window stable-capture loop shared by `wait stable` and the @@ -144,13 +141,7 @@ export async function runStableCaptureLoop( // loop recovers on private AX, the rest of the loop stays on it. let preferredBackend: SnapshotPreferredBackend | undefined; const pacing = createStableCapturePacing(start); - const adaptive = params.adaptiveConfirmation - ? createAdaptiveConfirmation({ - baseline: params.broadTransitionBaseline, - baselineFresh: params.adaptiveConfirmation.baselineFresh, - appBundleId: session?.appBundleId, - }) - : undefined; + const adaptive = await loadAdaptiveEarlyExit(params, session?.appBundleId); let captures = 0; let lastCapture: CapturedSnapshot | undefined; const result = ( @@ -175,14 +166,7 @@ export async function runStableCaptureLoop( runtime, options, deadlineMs - now(runtime), - params.broadTransitionBaseline - ? postActionPresentation(params.broadTransitionBaseline) - : { - ...postActionPresentation(undefined), - depth: options.depth, - scope: options.scope, - raw: options.raw, - }, + stableCapturePresentation(params.broadTransitionBaseline, options), preferredBackend, ); if (read.kind !== 'captured') return unfinished(read); @@ -219,25 +203,97 @@ export async function runStableCaptureLoop( } const step = recovered ? { settled: false, delayMs: recovered.delayMs } - : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures }); - if (step.settled) return result(true, false, nowMs - start); - const early = recovered - ? 'none' - : adaptive?.observe({ + : observeStableCapture({ + pacing, + adaptive, snapshot: capture.snapshot, signal, - floorInForce: requiredQuietMs > window.quietMs, + window, + nowMs, + deadlineMs, + captures, }); - if (early === 'settle') return result(true, false, nowMs - start); + if (step.settled) return result(true, false, nowMs - start); if (step.delayMs <= 0) break; - await sleep( - runtime, - early === 'confirm' ? Math.min(step.delayMs, ADAPTIVE_CONFIRMATION_GAP_MS) : step.delayMs, - ); + await sleep(runtime, step.delayMs); } return result(false, false, now(runtime) - start); } +/** + * The view every capture of the run reads: the baseline's, so the settled tree and the baseline + * compare (#4486), or without one the interactive view the caller's options narrow. + */ +function stableCapturePresentation( + baseline: SurfaceScopedNodes | undefined, + options: SelectorSnapshotOptions, +): SnapshotPresentation { + if (baseline) return postActionPresentation(baseline); + return { + ...postActionPresentation(undefined), + depth: options.depth, + scope: options.scope, + raw: options.raw, + }; +} + +/** + * The internal adaptive early exit, loaded only when a settle asks for it so the default path keeps + * its import closure. + */ +async function loadAdaptiveEarlyExit( + params: { + broadTransitionBaseline?: SurfaceScopedNodes; + adaptiveConfirmation?: { baselineFresh: boolean }; + }, + appBundleId: string | undefined, +): Promise { + if (!params.adaptiveConfirmation) return undefined; + const { createAdaptiveConfirmation } = await import('./settle-adaptive-confirmation.ts'); + return createAdaptiveConfirmation({ + baseline: params.broadTransitionBaseline, + baselineFresh: params.adaptiveConfirmation.baselineFresh, + appBundleId, + }); +} + +/** + * Folds one capture into the quiet window and, when the adaptive early exit is armed, into its + * short confirmation: settled, or how long to wait before the next capture (<= 0 ends the run). + */ +function observeStableCapture(params: { + pacing: StableCapturePacing; + adaptive: AdaptiveEarlyExit | undefined; + snapshot: SnapshotState; + signal: StableCaptureSignal; + window: ReturnType; + nowMs: number; + deadlineMs: number; + captures: number; +}): { settled: boolean; delayMs: number } { + const { window, adaptive } = params; + const step = params.pacing.observe({ + signal: params.signal, + nodeCount: params.snapshot.nodes.length, + nowMs: params.nowMs, + requiredQuietMs: window.requiredQuietMs, + pollMs: window.pollMs, + deadlineMs: params.deadlineMs, + captures: params.captures, + }); + if (step.settled || !adaptive) return step; + const early = adaptive.observe({ + snapshot: params.snapshot, + signal: params.signal, + floorInForce: window.requiredQuietMs > window.quietMs, + }); + if (early === 'settle') return { settled: true, delayMs: 0 }; + if (early === 'confirm' && step.delayMs > 0) { + return { settled: false, delayMs: Math.min(step.delayMs, adaptive.confirmGapMs) }; + } + return step; +} + /** * The quiet window one capture prices, the window it must actually hold once the broad-transition * confirmation has had its say, and the cadence that can measure that. Sampling an extended wait on diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 03c4c1cfe2..24abeb04b6 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -170,7 +170,12 @@ export type DaemonWaitTarget = kind: 'selector'; selector: string; timeoutMs?: number | null; - /** ADR 0012 / #1349: the recorded landmark identity a replayed wait must observe. */ + /** + * ADR 0012 / #1349, replay-only: the recorded landmark identity this wait must observe before + * reporting success. Polling is unchanged: the loop keeps waiting while no selector match + * carries this identity, and a timeout with rejected candidates throws the + * `WAIT_LANDMARK_MISMATCH_REASON` refusal instead of success. + */ recordedLandmark?: TargetAnnotationV1; } | { kind: 'absent'; selector: string; timeoutMs?: number | null } From 8d80138216850cfd6720f963556f96e5404ae2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 22:28:46 +0100 Subject: [PATCH 4/8] fix(settle): escalate the adaptive early exit on a confirmed broad replacement under any window The guard compared the required window with the requested one, so a requested window at or above the 1.5 s floor hid a confirmed broad replacement. The broad-transition confirmation now reports its latch, and the adaptive rule keys on that. Co-Authored-By: Claude Opus 5.5 --- .../settle-adaptive-confirmation.test.ts | 23 ++++++++++++++++++- .../runtime/settle-adaptive-confirmation.ts | 2 +- .../interaction/runtime/stable-capture.ts | 19 ++++++++++++--- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts index 5460b8e8ba..184b56946d 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts @@ -52,6 +52,7 @@ async function press(params: { stored?: SnapshotState; loop: SnapshotState[]; adaptive: boolean; + quietMs?: number; target?: Parameters['interactions']['press']>[0]; }) { let captures = 0; @@ -64,7 +65,10 @@ async function press(params: { }); const result = await device.interactions.press(params.target ?? ref('@e2'), { session: 'default', - settle: params.adaptive ? { earlyExit: 'adaptive', observationCurrent: true } : {}, + settle: { + ...(params.adaptive ? { earlyExit: 'adaptive' as const, observationCurrent: true } : {}), + ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }), + }, }); assert.ok(result.settle); return result.settle; @@ -160,6 +164,23 @@ test('a broad replacement owed the confirmation floor keeps the floor', async () assert.ok(adaptive.waitedMs >= 1_500, `adaptive lost the floor: ${adaptive.waitedMs}ms`); }); +test('a broad replacement under a requested window above the floor keeps that window', async () => { + // The floor raises nothing here, but the run still proved a broad replacement. + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [elementSettingsTabSnapshot()], + adaptive: true, + quietMs: 2_000, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok( + adaptive.waitedMs >= 2_000, + `adaptive lost the requested window: ${adaptive.waitedMs}ms`, + ); +}); + test('a point press whose baseline is the stored tree keeps the quiet window', async () => { // The pre-action evidence capture fails, so the baseline falls back to the stored tree, which may // predate a side effect: a change against it does not prove the action's effect. diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts index 3086787b27..58a9d3a67f 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -46,7 +46,7 @@ export function createAdaptiveConfirmation(params: { observe(input: { snapshot: SnapshotState; signal: StableCaptureSignal; - /** The broad-replacement floor raised this capture's window above the producer's own. */ + /** A capture proved a broad replacement, so the run owes the broad-replacement floor. */ floorInForce: boolean; }): 'settle' | 'confirm' | 'none' { if (!armed || !baseline) return 'none'; diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 5dd494cb6b..d869ef1151 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -285,7 +285,7 @@ function observeStableCapture(params: { const early = adaptive.observe({ snapshot: params.snapshot, signal: params.signal, - floorInForce: window.requiredQuietMs > window.quietMs, + floorInForce: window.floorConfirmed, }); if (early === 'settle') return { settled: true, delayMs: 0 }; if (early === 'confirm' && step.delayMs > 0) { @@ -303,10 +303,15 @@ function stableCaptureWindow( snapshot: SnapshotState, requestedQuietMs: number | undefined, confirmation: StableCaptureConfirmation, -): { quietMs: number; requiredQuietMs: number; pollMs: number } { +): { quietMs: number; requiredQuietMs: number; pollMs: number; floorConfirmed: boolean } { const quietMs = requestedQuietMs ?? defaultQuietMsForCapture(snapshot); const requiredQuietMs = confirmation.requiredQuietMs(snapshot, quietMs); - return { quietMs, requiredQuietMs, pollMs: stableCapturePollMs(requiredQuietMs) }; + return { + quietMs, + requiredQuietMs, + pollMs: stableCapturePollMs(requiredQuietMs), + floorConfirmed: confirmation.confirmed, + }; } /** @@ -488,6 +493,9 @@ function broadTransitionConfirmation(params: { ? currentQuietMs : Math.max(currentQuietMs, confirmedQuietMs); }, + get confirmed() { + return confirmedQuietMs !== undefined; + }, }; } @@ -497,6 +505,11 @@ type StableCaptureConfirmation = { snapshot: Parameters[0], currentQuietMs: number, ) => number; + /** + * A capture proved the broad replacement, so the floor holds for the rest of the run, whether or + * not it raised the window: a requested window above the floor still owes its full span. + */ + readonly confirmed: boolean; }; /** From 7442537119b8e558876335b35407b549150a4931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 16 Nov 2026 23:57:25 +0100 Subject: [PATCH 5/8] refactor(settle): leave the wait-target dedupe to its own change Co-Authored-By: Claude Opus 5.5 --- .../interaction/runtime/selector-wait.ts | 22 +++++++++++++++++-- .../contracts/src/daemon-command-surface.ts | 7 +----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts b/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts index ac0eb8dec3..2c7d4de739 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/selector-wait.ts @@ -1,6 +1,6 @@ import { AppError } from '@agent-device/kernel/errors'; import { findNodeByRef, normalizeRef, type SnapshotNode } from '@agent-device/kernel/snapshot'; -import type { DaemonWaitTarget } from '@agent-device/contracts/daemon-command-surface'; +import type { TargetAnnotationV1 } from '@agent-device/contracts/replay'; import type { PublicPlatform } from '@agent-device/kernel/device'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { checkWaitText, STALE_REF_HINT } from '@agent-device/selectors'; @@ -27,7 +27,25 @@ export type SelectorSnapshotOptions = { export type WaitCommandOptions = WaitCommandContext & SelectorSnapshotOptions & { - target: DaemonWaitTarget; + target: + | { kind: 'sleep'; durationMs: number } + | { kind: 'text'; text: string; timeoutMs?: number | null } + | { kind: 'ref'; ref: string; timeoutMs?: number | null } + | { + kind: 'selector'; + selector: string; + timeoutMs?: number | null; + /** + * ADR 0012 / #1349, replay-only: the recorded landmark identity this + * wait must observe before reporting success. Polling is unchanged — + * the loop keeps waiting while no selector match carries this + * identity, and a timeout with rejected candidates throws the + * `WAIT_LANDMARK_MISMATCH_REASON` refusal instead of success. + */ + recordedLandmark?: TargetAnnotationV1; + } + | { kind: 'absent'; selector: string; timeoutMs?: number | null } + | { kind: 'stable'; quietMs?: number | null; timeoutMs?: number | null }; }; export type WaitCommandResult = diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts index 24abeb04b6..03c4c1cfe2 100644 --- a/packages/contracts/src/daemon-command-surface.ts +++ b/packages/contracts/src/daemon-command-surface.ts @@ -170,12 +170,7 @@ export type DaemonWaitTarget = kind: 'selector'; selector: string; timeoutMs?: number | null; - /** - * ADR 0012 / #1349, replay-only: the recorded landmark identity this wait must observe before - * reporting success. Polling is unchanged: the loop keeps waiting while no selector match - * carries this identity, and a timeout with rejected candidates throws the - * `WAIT_LANDMARK_MISMATCH_REASON` refusal instead of success. - */ + /** ADR 0012 / #1349: the recorded landmark identity a replayed wait must observe. */ recordedLandmark?: TargetAnnotationV1; } | { kind: 'absent'; selector: string; timeoutMs?: number | null } From 5b81888eb47c4bc6d5c2474266373766acf34740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 00:14:41 +0100 Subject: [PATCH 6/8] fix(settle): escalate on a system-surface change and report the span an early exit held The adaptive early exit now keeps the quiet window when a capture reads another iOS system surface than the baseline, and the settle reports the quiet span its confirming pair held instead of the full window. Covers the adaptive switch on the press/click/longpress and generic routes, and the window fields surviving the opt-in branch. Co-Authored-By: Claude Opus 5.5 --- .../settle-adaptive-confirmation.test.ts | 14 ++++ .../runtime/settle-adaptive-confirmation.ts | 37 +++++++--- .../interaction/runtime/stable-capture.ts | 17 +++-- packages/contracts/src/interaction.ts | 4 + .../src/__tests__/support/store-factory.ts | 16 +++- .../src/execution/generic-settle.test.ts | 74 ++++++++++++++++++- .../internal/interaction-flags.test.ts | 11 ++- .../internal/interaction-settle.test.ts | 53 ++++++++++++- 8 files changed, 202 insertions(+), 24 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts index 184b56946d..60924e8abc 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts @@ -83,6 +83,9 @@ test('a changed capture that the next capture confirms settles before the quiet assert.equal(adaptive.settled, true); assert.equal(adaptive.captures, 2); assert.ok(adaptive.waitedMs < 200, `adaptive held ${adaptive.waitedMs}ms`); + // The run reports the span the confirming pair held, not the window it never waited out. + assert.equal(adaptive.quietMs, adaptive.waitedMs); + assert.ok(quiet.quietMs >= DEFAULT_QUIET_MS); assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!'))); assert.equal(quiet.settled, true); assert.ok(quiet.waitedMs >= DEFAULT_QUIET_MS, `quiet window held ${quiet.waitedMs}ms`); @@ -281,3 +284,14 @@ for (const [label, observationCurrent, early] of [ assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`); }); } + +test('a capture on an iOS system surface the baseline did not show keeps the quiet window', async () => { + // An app action can raise an in-place system surface (a web sign-in sheet, #2438); two equal + // captures of it are not the app settling. + const sheet = { ...relabeled(), iosSystemSurfaceBundleId: 'com.apple.SafariViewService' }; + + const adaptive = await press({ loop: [sheet], adaptive: true }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts index 58a9d3a67f..7141420d31 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -33,43 +33,47 @@ export function createAdaptiveConfirmation(params: { }) { const { baseline, appBundleId } = params; let armed = params.baselineFresh && baseline !== undefined && baseline.nodes.length > 0; - let candidate: { signal: StableCaptureSignal; snapshot: SnapshotState } | undefined; + let candidate: { signal: StableCaptureSignal; snapshot: SnapshotState; atMs: number } | undefined; const disarm = () => { armed = false; candidate = undefined; }; return { /** - * Folds one capture in. `settle`: this capture confirmed the previous one. `confirm`: take the - * next capture after the short gap. `none`: the quiet window decides. + * Folds one capture in. `settle`: this capture confirmed the previous one, and the pair held + * still for `heldQuietMs`. `confirm`: take the next capture after the short gap. `none`: the + * quiet window decides. */ observe(input: { snapshot: SnapshotState; signal: StableCaptureSignal; + nowMs: number; /** A capture proved a broad replacement, so the run owes the broad-replacement floor. */ floorInForce: boolean; - }): 'settle' | 'confirm' | 'none' { - if (!armed || !baseline) return 'none'; + }): { kind: 'settle'; heldQuietMs: number } | { kind: 'confirm' } | { kind: 'none' } { + if (!armed || !baseline) return { kind: 'none' }; const { snapshot, signal } = input; if ( input.floorInForce || hasBusyIndicator(snapshot) || isTransitionalUnion(baseline, snapshot, appBundleId) || - presentedSurfaceChanged(baseline, snapshot) + presentedSurfaceChanged(baseline, snapshot) || + surfaceBundleChanged(baseline, snapshot) ) { disarm(); - return 'none'; + return { kind: 'none' }; } if (candidate) { const confirmed = sameProvenance(candidate.snapshot, snapshot) && stableCaptureSignalsEqual(candidate.signal, signal); + const heldQuietMs = input.nowMs - candidate.atMs; disarm(); - return confirmed ? 'settle' : 'none'; + return confirmed ? { kind: 'settle', heldQuietMs } : { kind: 'none' }; } - if (!differsFromBaseline(baseline, snapshot, appBundleId)) return 'none'; - candidate = { signal, snapshot }; - return 'confirm'; + if (!differsFromBaseline(baseline, snapshot, appBundleId)) return { kind: 'none' }; + candidate = { signal, snapshot, atMs: input.nowMs }; + return { kind: 'confirm' }; }, /** A private-AX recovery re-read the screen through another backend. */ disarm, @@ -135,10 +139,19 @@ function presentedSurfaceChanged(baseline: SurfaceScopedNodes, snapshot: Snapsho return before !== undefined && after !== undefined && before !== after; } +/** + * The capture reads another surface than the baseline: an in-place iOS system surface (a web sign-in + * sheet) arriving over the app, or leaving it (#2438). An app capture names none. + */ +function surfaceBundleChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean { + return baseline.surfaceBundleId !== snapshot.iosSystemSurfaceBundleId; +} + function sameProvenance(left: SnapshotState, right: SnapshotState): boolean { return ( left.producer === right.producer && left.backend === right.backend && - left.presentedSurface === right.presentedSurface + left.presentedSurface === right.presentedSurface && + left.iosSystemSurfaceBundleId === right.iosSystemSurfaceBundleId ); } diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index d869ef1151..03caee7252 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -85,7 +85,8 @@ export type StableCaptureLoopResult = { /** * The quiet window the run was holding when it returned: the caller's, or the default resolved * from the last capture that paid it. A run that completed no capture never priced a window, so - * it reports the general default it started from. + * it reports the general default it started from. A run the adaptive early exit settled reports + * the span its confirming pair actually held instead. */ quietMs: number; /** The most recent completed capture, settled or not. */ @@ -213,7 +214,10 @@ export async function runStableCaptureLoop( deadlineMs, captures, }); - if (step.settled) return result(true, false, nowMs - start); + if (step.settled) { + quietMs = step.heldQuietMs ?? quietMs; + return result(true, false, nowMs - start); + } if (step.delayMs <= 0) break; await sleep(runtime, step.delayMs); } @@ -270,7 +274,7 @@ function observeStableCapture(params: { nowMs: number; deadlineMs: number; captures: number; -}): { settled: boolean; delayMs: number } { +}): { settled: boolean; delayMs: number; heldQuietMs?: number } { const { window, adaptive } = params; const step = params.pacing.observe({ signal: params.signal, @@ -285,10 +289,13 @@ function observeStableCapture(params: { const early = adaptive.observe({ snapshot: params.snapshot, signal: params.signal, + nowMs: params.nowMs, floorInForce: window.floorConfirmed, }); - if (early === 'settle') return { settled: true, delayMs: 0 }; - if (early === 'confirm' && step.delayMs > 0) { + if (early.kind === 'settle') { + return { settled: true, delayMs: 0, heldQuietMs: early.heldQuietMs }; + } + if (early.kind === 'confirm' && step.delayMs > 0) { return { settled: false, delayMs: Math.min(step.delayMs, adaptive.confirmGapMs) }; } return step; diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts index 8f0b5327bd..97d6198c4d 100644 --- a/packages/contracts/src/interaction.ts +++ b/packages/contracts/src/interaction.ts @@ -331,6 +331,10 @@ export type SettleObservation = { settled: boolean; waitedMs: number; captures: number; + /** + * The quiet window the settle held: the requested or producer default window, or, when the + * internal adaptive early exit settled the run, the span its confirming pair actually held. + */ quietMs: number; timeoutMs: number; /** diff --git a/packages/daemon/src/__tests__/support/store-factory.ts b/packages/daemon/src/__tests__/support/store-factory.ts index cf2efaa757..5ea996ebb8 100644 --- a/packages/daemon/src/__tests__/support/store-factory.ts +++ b/packages/daemon/src/__tests__/support/store-factory.ts @@ -1,6 +1,10 @@ import path from 'node:path'; import { SessionStore } from '../../core/session-store.ts'; -import { createLiveTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + createLiveTestHostEnvironment, + createTestHostEnvironment, + mkdtempForTestSync, +} from '@agent-device/test-kit'; import type { SessionRef, SessionState } from '../../core/session-state.ts'; export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore { @@ -8,6 +12,16 @@ export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore { return new SessionStore(path.join(tempRoot, 'sessions'), createLiveTestHostEnvironment()); } +/** A store whose daemon host environment adds `env`, such as an internal opt-in switch. */ +export function makeSessionStoreWithEnv(env: Record): SessionStore { + const tempRoot = mkdtempForTestSync('agent-device-test-'); + const host = createTestHostEnvironment(); + return new SessionStore(path.join(tempRoot, 'sessions'), { + ...host, + env: { ...host.env, ...env }, + }); +} + export function makeStoredSessionRef(session: SessionState, address = session.name): SessionRef { return makeSessionStore().publish(address, session); } diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts index f301e83e36..37ad70a295 100644 --- a/packages/daemon/src/execution/generic-settle.test.ts +++ b/packages/daemon/src/execution/generic-settle.test.ts @@ -7,7 +7,7 @@ import type { CommandFlags } from '@agent-device/contracts/command'; import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot'; import type { IosSystemSurfaceProvenance } from '@agent-device/contracts/ios-system-surface'; import { makeIosSession } from '../__tests__/support/session-factories.ts'; -import { makeSessionStore } from '../__tests__/support/store-factory.ts'; +import { makeSessionStore, makeSessionStoreWithEnv } from '../__tests__/support/store-factory.ts'; import { activateCompleteRefFrame, markSessionSnapshotOutdated, @@ -712,3 +712,75 @@ for (const change of ['rebuild', 'retire'] as const) { } }); } + +const ADAPTIVE_ENV = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }; + +test('back --settle with the adaptive switch arms a current stored tree as a fresh baseline', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-current'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]).toMatchObject({ + quietMs: 25, + timeoutMs: 2_000, + earlyExit: 'adaptive', + observationCurrent: true, + baselineFresh: true, + }); +}); + +test('back --settle with the adaptive switch does not trust a stored tree a native read outdated', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-outdated'; + const session = seedSession(sessionName, sessionStore); + markSessionSnapshotOutdated(session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]).toMatchObject({ + earlyExit: 'adaptive', + observationCurrent: false, + baselineFresh: false, + }); +}); + +test('back --settle with the adaptive switch does not trust a baseline the frame answered', async () => { + const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV); + const sessionName = 'generic-settle-adaptive-frame'; + // The frame was read interactive-only; a later full read replaced the stored tree, so the frame + // answers the baseline and a newer observation exists. + const session = makeIosSession(sessionName); + setSessionSnapshot( + session, + buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES }), { snapshotInteractiveOnly: true }), + ); + activateCompleteRefFrame(session); + setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {})); + sessionStore.publish(sessionName, session); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]?.baseline.presentation?.interactiveOnly).toBe(true); + expect(settleObservationCalls[0]).toMatchObject({ + observationCurrent: false, + baselineFresh: false, + }); +}); + +test('back --settle without the adaptive switch forwards no early exit', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'generic-settle-adaptive-off'; + seedSession(sessionName, sessionStore); + mockCommandDispatch([AFTER_NODES]); + + await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } }); + + expect(settleObservationCalls[0]?.earlyExit).toBeUndefined(); + expect(settleObservationCalls[0]?.observationCurrent).toBeUndefined(); + expect(settleObservationCalls[0]?.baselineFresh).toBe(false); +}); diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts index 4bb7d0c2cb..4b50808476 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts @@ -23,10 +23,13 @@ function sessionWithFrame() { test('the host environment opts a settle request into the adaptive confirmation', () => { const session = sessionWithFrame(); - assert.deepEqual(readSettleRequest({ settle: true }, { env: OPT_IN, session }), { - earlyExit: 'adaptive', - observationCurrent: true, - }); + assert.deepEqual( + readSettleRequest( + { settle: true, settleQuietMs: 700, timeoutMs: 4_000 }, + { env: OPT_IN, session }, + ), + { quietMs: 700, timeoutMs: 4_000, earlyExit: 'adaptive', observationCurrent: true }, + ); }); test('a ref frame a side effect crossed is not current for the adaptive confirmation', () => { diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts index 0cd73501b6..bc1efd50c3 100644 --- a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts +++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts @@ -12,7 +12,10 @@ import { refFrameState, refFrameTree, } from '../../../core/ref-frame.ts'; -import { makeSessionStore } from '../../../__tests__/support/store-factory.ts'; +import { + makeSessionStore, + makeSessionStoreWithEnv, +} from '../../../__tests__/support/store-factory.ts'; import { makeIosSession } from '../../../__tests__/support/session-factories.ts'; import { clearRequestAbortRegistration, @@ -682,3 +685,51 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn expect(settle.settled).toBe(true); expect(typeof settle.refsGeneration).toBe('number'); }); + +// hover is web-only (refused on this iOS session); it reads the same shared settle options. +for (const command of ['press', 'click', 'longpress'] as const) { + for (const adaptive of [true, false]) { + test(`${command} --settle forwards ${adaptive ? 'the adaptive early exit' : 'only its window'} to the settle observation`, async () => { + const sessionStore = adaptive + ? makeSessionStoreWithEnv({ AGENT_DEVICE_SETTLE_ADAPTIVE: '1' }) + : makeSessionStore(); + const sessionName = `settle-forward-${command}-${adaptive}`; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES] }); + const forwarded: unknown[] = []; + const recordingSurface = createTouchCommandSurface({ + observeSettle: async ({ settle }) => { + forwarded.push(settle); + return { settled: true, waitedMs: 0, captures: 1, quietMs: 25, timeoutMs: 2_000 }; + }, + }); + + const response = await handleInteractionCommands({ + createCommandSurface: recordingSurface, + req: { + token: 't', + session: sessionName, + command, + positionals: ['label=Continue'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + captureSnapshotForSession: mockCaptureSnapshotForSession, + ...getRuntimeBindings(), + }); + + expectOkData(response); + // The seeded snapshot published a complete frame and nothing outdated it, so the observation + // is current before the action. + expect(forwarded).toEqual([ + { + quietMs: 25, + timeoutMs: 2_000, + ...(adaptive ? { earlyExit: 'adaptive', observationCurrent: true } : {}), + }, + ]); + }); + } +} From 7dad132c87ea999d829ac3f43ea07aeb3f7f4cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 17 Nov 2026 00:14:41 +0100 Subject: [PATCH 7/8] test(test-app): acknowledge each settle lab effect at once and drop fired timers Co-Authored-By: Claude Opus 5.5 --- examples/test-app/src/screens/SettleLabScreen.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/test-app/src/screens/SettleLabScreen.tsx b/examples/test-app/src/screens/SettleLabScreen.tsx index 3b2bd1fd9d..d04beadccd 100644 --- a/examples/test-app/src/screens/SettleLabScreen.tsx +++ b/examples/test-app/src/screens/SettleLabScreen.tsx @@ -4,8 +4,9 @@ import { ActivityIndicator, Animated, Pressable, StyleSheet, Text, View } from ' /** * Timed UI effects for measuring `--settle`: each button starts one effect whose final screen is - * known, so a settled tree can be checked against it. `settle-status` names the final state of the - * last effect with its run number. + * known, so a settled tree can be checked against it. `settle-run` acknowledges each effect at once, + * the way an app shows a pending state, and `settle-status` names the final state of the last effect + * with its run number. */ const DELAYS_MS = [500, 800] as const; const FADE_MS = 750; @@ -25,7 +26,11 @@ export function SettleLabScreen() { useEffect(() => () => timers.current.forEach(clearTimeout), []); const later = (ms: number, effect: () => void) => { - timers.current.push(setTimeout(effect, ms)); + const handle = setTimeout(() => { + timers.current = timers.current.filter((pending) => pending !== handle); + effect(); + }, ms); + timers.current.push(handle); }; const nextRun = () => { const next = run + 1; @@ -42,6 +47,9 @@ export function SettleLabScreen() { {`Count ${count}`} + + {`Run ${run}`} + setCount((c) => c + 1)} /> {DELAYS_MS.map((delayMs) => ( Date: Tue, 17 Nov 2026 00:25:03 +0100 Subject: [PATCH 8/8] fix(settle): hold the adaptive early exit while an armed latch has not judged a broad replacement A rootless read after a non-presented baseline fails the broad-transition gate, so the latch stays an open question that a later rooted capture can still confirm. The adaptive rule now asks the shared broad-replacement predicate against the baseline in that state and escalates. Co-Authored-By: Claude Opus 5.5 --- .../settle-adaptive-confirmation.test.ts | 17 +++++++++++++++ .../runtime/settle-adaptive-confirmation.ts | 2 +- .../interaction/runtime/stable-capture.ts | 21 ++++++++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts index 60924e8abc..cb62137296 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts @@ -295,3 +295,20 @@ test('a capture on an iOS system surface the baseline did not show keeps the qui assert.equal(adaptive.settled, true); assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); }); + +test('a rootless broad replacement the latch could not judge yet keeps the quiet window', async () => { + // A rootless read after a non-presented baseline fails the latch gate, so the floor stays an open + // question; a later rooted capture could still confirm it, so two rootless reads must not settle. + const rootless = elementSettingsTabSnapshot(); + rootless.nodes = rootless.nodes.filter((node) => node.type !== 'Application'); + + const adaptive = await press({ + stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' }, + loop: [rootless], + adaptive: true, + target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`), + }); + + assert.equal(adaptive.settled, true); + assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`); +}); diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts index 7141420d31..18030d75af 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts @@ -48,7 +48,7 @@ export function createAdaptiveConfirmation(params: { snapshot: SnapshotState; signal: StableCaptureSignal; nowMs: number; - /** A capture proved a broad replacement, so the run owes the broad-replacement floor. */ + /** The run owes, or may yet owe, the broad-replacement floor for this capture. */ floorInForce: boolean; }): { kind: 'settle'; heldQuietMs: number } | { kind: 'confirm' } | { kind: 'none' } { if (!armed || !baseline) return { kind: 'none' }; diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts index 03caee7252..de7ed7665e 100644 --- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts +++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts @@ -207,6 +207,7 @@ export async function runStableCaptureLoop( : observeStableCapture({ pacing, adaptive, + confirmation, snapshot: capture.snapshot, signal, window, @@ -268,6 +269,7 @@ async function loadAdaptiveEarlyExit( function observeStableCapture(params: { pacing: StableCapturePacing; adaptive: AdaptiveEarlyExit | undefined; + confirmation: StableCaptureConfirmation; snapshot: SnapshotState; signal: StableCaptureSignal; window: ReturnType; @@ -290,7 +292,8 @@ function observeStableCapture(params: { snapshot: params.snapshot, signal: params.signal, nowMs: params.nowMs, - floorInForce: window.floorConfirmed, + floorInForce: + window.floorConfirmed || params.confirmation.unresolvedReplacement(params.snapshot), }); if (early.kind === 'settle') { return { settled: true, delayMs: 0, heldQuietMs: early.heldQuietMs }; @@ -503,6 +506,16 @@ function broadTransitionConfirmation(params: { get confirmed() { return confirmedQuietMs !== undefined; }, + unresolvedReplacement(snapshot) { + // The gate has refused every capture so far, so the latch has not ruled; a later capture it + // admits could still confirm the floor. Judge this capture against the baseline directly. + if (!baseline || transitionBaseline !== undefined) return false; + if (!isBoundIosXCTestCapture(snapshot.backend, platform)) return false; + return stableCaptureSignalsHaveBroadReplacement( + stableCaptureComparisonSignal(snapshot, baseline.nodes, appBundleId), + stableCaptureComparisonSignal(snapshot, snapshot.nodes, appBundleId), + ); + }, }; } @@ -517,6 +530,12 @@ type StableCaptureConfirmation = { * not it raised the window: a requested window above the floor still owes its full span. */ readonly confirmed: boolean; + /** + * The latch is armed but no capture has passed its gate yet (a rootless read after a + * non-presented baseline), and this capture already replaces the baseline broadly. Asked only by + * the adaptive early exit, which must not settle before a later capture could confirm the floor. + */ + unresolvedReplacement: (snapshot: Parameters[0]) => boolean; }; /**