;
@@ -78,6 +80,7 @@ export function createSettleDevice(params: {
{
name: 'default',
snapshot: params.stored,
+ ...(params.refFrame ? { refFrameSnapshot: params.refFrame } : {}),
...(params.appBundleId ? { appBundleId: params.appBundleId } : {}),
},
]),
diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts
new file mode 100644
index 0000000000..cb62137296
--- /dev/null
+++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.test.ts
@@ -0,0 +1,314 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import type { SnapshotProvenance, SnapshotState } from '@agent-device/contracts/snapshot-model';
+import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
+import { ref, selector } from './selector-read-utils.ts';
+import { createSettleDevice, welcomeSnapshot } from './__tests__/settle-device-fixtures.ts';
+import {
+ elementCatalogTabSnapshot,
+ elementSettingsTabSnapshot,
+} from './stable-capture.fixtures.ts';
+
+// The internal adaptive confirmation prototype: a capture that differs from a fresh pre-action
+// baseline settles once one short confirmation agrees with it; every escalation keeps the quiet
+// window. Each case runs the same capture sequence with the switch on and off, so the difference
+// is the rule's and not the fixture's. Budgets run on the injected clock; captures cost nothing.
+
+const DEFAULT_QUIET_MS = 500;
+
+/** The welcome screen after a press relabels its heading in place: no broad replacement. */
+function relabeled(label = 'Welcome back!'): SnapshotState {
+ const base = welcomeSnapshot();
+ return {
+ ...base,
+ nodes: base.nodes.map((node) => (node.index === 0 ? { ...node, label } : node)),
+ };
+}
+
+function withProvenance(state: SnapshotState, provenance: P) {
+ return { ...state, ...provenance };
+}
+
+function withSpinner(snapshot: SnapshotState): SnapshotState {
+ return makeSnapshotState([
+ ...snapshot.nodes,
+ {
+ index: snapshot.nodes.length,
+ depth: 1,
+ parentIndex: 0,
+ type: 'ActivityIndicator',
+ label: 'In progress',
+ rect: { x: 10, y: 400, width: 20, height: 20 },
+ },
+ ]);
+}
+
+/**
+ * An @ref press on the stored tree, whose loop captures follow `loop` (the last one repeats). The
+ * native ref route captures nothing before the action, so the stored tree is the baseline, fresh
+ * here because the daemon reports that observation current.
+ */
+async function press(params: {
+ stored?: SnapshotState;
+ loop: SnapshotState[];
+ adaptive: boolean;
+ quietMs?: number;
+ target?: Parameters['interactions']['press']>[0];
+}) {
+ let captures = 0;
+ const device = createSettleDevice({
+ stored: params.stored ?? welcomeSnapshot(),
+ captureSnapshot: () => {
+ captures += 1;
+ return { snapshot: params.loop[Math.min(captures, params.loop.length) - 1]! };
+ },
+ });
+ const result = await device.interactions.press(params.target ?? ref('@e2'), {
+ session: 'default',
+ settle: {
+ ...(params.adaptive ? { earlyExit: 'adaptive' as const, observationCurrent: true } : {}),
+ ...(params.quietMs === undefined ? {} : { quietMs: params.quietMs }),
+ },
+ });
+ assert.ok(result.settle);
+ return result.settle;
+}
+
+test('a changed capture that the next capture confirms settles before the quiet window', async () => {
+ const loop = [relabeled(), relabeled()];
+
+ const adaptive = await press({ loop, adaptive: true });
+ const quiet = await press({ loop, adaptive: false });
+
+ assert.equal(adaptive.settled, true);
+ assert.equal(adaptive.captures, 2);
+ assert.ok(adaptive.waitedMs < 200, `adaptive held ${adaptive.waitedMs}ms`);
+ // The run reports the span the confirming pair held, not the window it never waited out.
+ assert.equal(adaptive.quietMs, adaptive.waitedMs);
+ assert.ok(quiet.quietMs >= DEFAULT_QUIET_MS);
+ assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!')));
+ assert.equal(quiet.settled, true);
+ assert.ok(quiet.waitedMs >= DEFAULT_QUIET_MS, `quiet window held ${quiet.waitedMs}ms`);
+});
+
+test('a confirmation that disagrees falls back to the full quiet window', async () => {
+ const adaptive = await press({
+ loop: [relabeled('Loading'), relabeled('Welcome back!')],
+ adaptive: true,
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+ assert.ok(adaptive.diff?.lines.some((line) => line.text.includes('Welcome back!')));
+});
+
+test('a capture that still matches the baseline keeps the quiet window', async () => {
+ const adaptive = await press({ loop: [welcomeSnapshot()], adaptive: true });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a change that arrives after an unchanged capture is confirmed on its own', async () => {
+ const adaptive = await press({
+ loop: [welcomeSnapshot(), relabeled(), relabeled()],
+ adaptive: true,
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.equal(adaptive.captures, 3);
+ assert.ok(adaptive.waitedMs < DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a busy indicator on the changed screen keeps the quiet window', async () => {
+ const loop = [withSpinner(relabeled())];
+
+ const adaptive = await press({ loop, adaptive: true });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a confirmation from another producer keeps the quiet window', async () => {
+ const adaptive = await press({
+ loop: [
+ withProvenance(relabeled(), { backend: 'xctest', producer: 'apple-runner' }),
+ withProvenance(relabeled(), { backend: 'xctest', producer: 'appium-source' }),
+ ],
+ adaptive: true,
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a presented surface the baseline did not show keeps the quiet window', async () => {
+ const adaptive = await press({
+ stored: { ...welcomeSnapshot(), presentedSurface: 'absent' },
+ loop: [{ ...relabeled(), presentedSurface: 'presented' }],
+ adaptive: true,
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a broad replacement owed the confirmation floor keeps the floor', async () => {
+ const settings = elementSettingsTabSnapshot();
+
+ const adaptive = await press({
+ stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' },
+ loop: [settings],
+ adaptive: true,
+ target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`),
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= 1_500, `adaptive lost the floor: ${adaptive.waitedMs}ms`);
+});
+
+test('a broad replacement under a requested window above the floor keeps that window', async () => {
+ // The floor raises nothing here, but the run still proved a broad replacement.
+ const adaptive = await press({
+ stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' },
+ loop: [elementSettingsTabSnapshot()],
+ adaptive: true,
+ quietMs: 2_000,
+ target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`),
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(
+ adaptive.waitedMs >= 2_000,
+ `adaptive lost the requested window: ${adaptive.waitedMs}ms`,
+ );
+});
+
+test('a point press whose baseline is the stored tree keeps the quiet window', async () => {
+ // The pre-action evidence capture fails, so the baseline falls back to the stored tree, which may
+ // predate a side effect: a change against it does not prove the action's effect.
+ let captures = 0;
+ const device = createSettleDevice({
+ stored: welcomeSnapshot(),
+ captureSnapshot: () => {
+ captures += 1;
+ if (captures === 1) throw new Error('capture unavailable');
+ return { snapshot: relabeled() };
+ },
+ });
+
+ const result = await device.interactions.press(
+ { kind: 'point', x: 50, y: 100 },
+ { session: 'default', settle: { earlyExit: 'adaptive' } },
+ );
+
+ assert.equal(result.settle?.settled, true);
+ assert.ok(result.settle.waitedMs >= DEFAULT_QUIET_MS, `held ${result.settle.waitedMs}ms`);
+});
+
+test('a capture that holds the baseline beside a new screen keeps the quiet window', async () => {
+ // A navigation transition can publish both screens at once and hold that union across captures.
+ const base = welcomeSnapshot();
+ const union = makeSnapshotState([
+ ...base.nodes,
+ ...['Detail', 'Buy', 'Share'].map((label, offset) => ({
+ index: base.nodes.length + offset,
+ depth: 1,
+ parentIndex: 0,
+ type: 'Button',
+ label,
+ rect: { x: 200, y: 20 + offset * 60, width: 100, height: 40 },
+ })),
+ ]);
+
+ const adaptive = await press({ loop: [union], adaptive: true });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+for (const [label, observationCurrent, early] of [
+ ['a current observation is a fresh baseline', true, true],
+ ['a frame the daemon reports outdated keeps the quiet window', false, false],
+] as const) {
+ test(`a selector press read in another view than the frame: ${label}`, async () => {
+ // The resolution capture carries no presentation, the frame is interactive-only, so the settle
+ // baseline is the frame itself and only the daemon can say whether it is current.
+ const frame = { ...welcomeSnapshot(), presentation: { interactiveOnly: true } };
+ let captures = 0;
+ const device = createSettleDevice({
+ stored: frame,
+ refFrame: frame,
+ captureSnapshot: () => {
+ captures += 1;
+ return { snapshot: captures === 1 ? welcomeSnapshot() : relabeled() };
+ },
+ });
+
+ const result = await device.interactions.press(selector('label=Next'), {
+ session: 'default',
+ settle: { earlyExit: 'adaptive', observationCurrent },
+ });
+
+ assert.equal(result.settle?.settled, true);
+ assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`);
+ });
+}
+
+for (const [label, observationCurrent, early] of [
+ ['a current observation is a fresh baseline', true, true],
+ ['a frame the daemon reports outdated keeps the quiet window', false, false],
+] as const) {
+ test(`an @ref press on the stored frame: ${label}`, async () => {
+ // The ref route resolves against the stored frame and captures nothing before the action, so the
+ // frame is the baseline. After `wait text` (or an Android freshness read) the frame stays active
+ // but no longer describes the screen, and only the daemon knows that.
+ const frame = welcomeSnapshot();
+ let captures = 0;
+ const device = createSettleDevice({
+ stored: frame,
+ refFrame: frame,
+ captureSnapshot: () => {
+ captures += 1;
+ return { snapshot: relabeled() };
+ },
+ });
+
+ const result = await device.interactions.press(ref('@e2'), {
+ session: 'default',
+ settle: { earlyExit: 'adaptive', observationCurrent },
+ });
+
+ assert.equal(captures > 0, true);
+ assert.equal(result.settle?.settled, true);
+ assert.equal(result.settle.waitedMs < DEFAULT_QUIET_MS, early, `${result.settle.waitedMs}ms`);
+ });
+}
+
+test('a capture on an iOS system surface the baseline did not show keeps the quiet window', async () => {
+ // An app action can raise an in-place system surface (a web sign-in sheet, #2438); two equal
+ // captures of it are not the app settling.
+ const sheet = { ...relabeled(), iosSystemSurfaceBundleId: 'com.apple.SafariViewService' };
+
+ const adaptive = await press({ loop: [sheet], adaptive: true });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
+
+test('a rootless broad replacement the latch could not judge yet keeps the quiet window', async () => {
+ // A rootless read after a non-presented baseline fails the latch gate, so the floor stays an open
+ // question; a later rooted capture could still confirm it, so two rootless reads must not settle.
+ const rootless = elementSettingsTabSnapshot();
+ rootless.nodes = rootless.nodes.filter((node) => node.type !== 'Application');
+
+ const adaptive = await press({
+ stored: { ...elementCatalogTabSnapshot(), presentedSurface: 'unknown' },
+ loop: [rootless],
+ adaptive: true,
+ target: ref(`@${elementCatalogTabSnapshot().nodes.find((node) => node.ref)!.ref}`),
+ });
+
+ assert.equal(adaptive.settled, true);
+ assert.ok(adaptive.waitedMs >= DEFAULT_QUIET_MS, `adaptive held ${adaptive.waitedMs}ms`);
+});
diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts
new file mode 100644
index 0000000000..18030d75af
--- /dev/null
+++ b/packages/command-runtime/src/commands/interaction/runtime/settle-adaptive-confirmation.ts
@@ -0,0 +1,157 @@
+import { isBusyIndicatorNode } from '@agent-device/kernel/snapshot';
+import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
+import type { SurfaceScopedNodes } from '@agent-device/contracts/interaction';
+import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome';
+import {
+ sharedIdentityCount,
+ stableCaptureSignal,
+ stableCaptureSignalsEqual,
+ type StableCaptureSignal,
+} from './stable-capture-signal.ts';
+
+/**
+ * Internal prototype (`SettleParams.earlyExit: 'adaptive'`): a capture that differs from a fresh
+ * pre-action baseline, and that one short confirmation capture agrees with, settles before the
+ * quiet window. Every other case keeps the quiet window: no capture has differed yet (a no-op, a
+ * toggle that has not rendered, a late effect), the confirmation disagrees, a busy indicator is on
+ * screen, the producer or the presented surface changed, the broad-replacement floor is in force,
+ * the capture still holds the baseline screen beside a new one, or the baseline may predate a side
+ * effect. One failed check disarms the rule for the rest of the run.
+ *
+ * Two equal captures ~110 ms apart cannot exclude a later stage nothing marks: a toast standing in
+ * for navigation, a plateau between transition stages, a late value write. That is the trade this
+ * prototype measures.
+ */
+const ADAPTIVE_CONFIRMATION_GAP_MS = 110;
+
+export type AdaptiveEarlyExit = ReturnType;
+
+export function createAdaptiveConfirmation(params: {
+ baseline: SurfaceScopedNodes | undefined;
+ baselineFresh: boolean;
+ appBundleId: string | undefined;
+}) {
+ const { baseline, appBundleId } = params;
+ let armed = params.baselineFresh && baseline !== undefined && baseline.nodes.length > 0;
+ let candidate: { signal: StableCaptureSignal; snapshot: SnapshotState; atMs: number } | undefined;
+ const disarm = () => {
+ armed = false;
+ candidate = undefined;
+ };
+ return {
+ /**
+ * Folds one capture in. `settle`: this capture confirmed the previous one, and the pair held
+ * still for `heldQuietMs`. `confirm`: take the next capture after the short gap. `none`: the
+ * quiet window decides.
+ */
+ observe(input: {
+ snapshot: SnapshotState;
+ signal: StableCaptureSignal;
+ nowMs: number;
+ /** The run owes, or may yet owe, the broad-replacement floor for this capture. */
+ floorInForce: boolean;
+ }): { kind: 'settle'; heldQuietMs: number } | { kind: 'confirm' } | { kind: 'none' } {
+ if (!armed || !baseline) return { kind: 'none' };
+ const { snapshot, signal } = input;
+ if (
+ input.floorInForce ||
+ hasBusyIndicator(snapshot) ||
+ isTransitionalUnion(baseline, snapshot, appBundleId) ||
+ presentedSurfaceChanged(baseline, snapshot) ||
+ surfaceBundleChanged(baseline, snapshot)
+ ) {
+ disarm();
+ return { kind: 'none' };
+ }
+ if (candidate) {
+ const confirmed =
+ sameProvenance(candidate.snapshot, snapshot) &&
+ stableCaptureSignalsEqual(candidate.signal, signal);
+ const heldQuietMs = input.nowMs - candidate.atMs;
+ disarm();
+ return confirmed ? { kind: 'settle', heldQuietMs } : { kind: 'none' };
+ }
+ if (!differsFromBaseline(baseline, snapshot, appBundleId)) return { kind: 'none' };
+ candidate = { signal, snapshot, atMs: input.nowMs };
+ return { kind: 'confirm' };
+ },
+ /** A private-AX recovery re-read the screen through another backend. */
+ disarm,
+ /** The wait before the confirming capture. */
+ confirmGapMs: ADAPTIVE_CONFIRMATION_GAP_MS,
+ };
+}
+
+/**
+ * A navigation transition can publish both screens at once, and that union can hold still across
+ * two captures. A capture that keeps most of the baseline while adding half as much again is that
+ * union, not a settled screen.
+ */
+const UNION_RETAINED_RATIO = 0.8;
+const UNION_GROWTH_RATIO = 0.5;
+
+function isTransitionalUnion(
+ baseline: SurfaceScopedNodes,
+ snapshot: SnapshotState,
+ appBundleId: string | undefined,
+): boolean {
+ const before = projectedSignal(snapshot, baseline.nodes, appBundleId);
+ const after = projectedSignal(snapshot, snapshot.nodes, appBundleId);
+ if (before.nodes.length === 0) return false;
+ const shared = sharedIdentityCount(before, after);
+ return (
+ shared / before.nodes.length >= UNION_RETAINED_RATIO &&
+ (after.nodes.length - shared) / before.nodes.length >= UNION_GROWTH_RATIO
+ );
+}
+
+/**
+ * One side of a baseline comparison: both sides take the capture's provenance so visibility
+ * filtering applies alike, and both drop the keyboard chrome the settle diff collapses.
+ */
+function projectedSignal(
+ snapshot: SnapshotState,
+ nodes: SnapshotState['nodes'],
+ appBundleId: string | undefined,
+): StableCaptureSignal {
+ return stableCaptureSignal({ ...snapshot, nodes: withoutSettleChrome(nodes, appBundleId) });
+}
+
+function differsFromBaseline(
+ baseline: SurfaceScopedNodes,
+ snapshot: SnapshotState,
+ appBundleId: string | undefined,
+): boolean {
+ // A keyboard alone is not the action's effect: the projection drops it.
+ return !stableCaptureSignalsEqual(
+ projectedSignal(snapshot, baseline.nodes, appBundleId),
+ projectedSignal(snapshot, snapshot.nodes, appBundleId),
+ );
+}
+
+function hasBusyIndicator(snapshot: SnapshotState): boolean {
+ return snapshot.nodes.some(isBusyIndicatorNode);
+}
+
+function presentedSurfaceChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean {
+ const before = baseline.presentedSurface;
+ const after = snapshot.presentedSurface;
+ return before !== undefined && after !== undefined && before !== after;
+}
+
+/**
+ * The capture reads another surface than the baseline: an in-place iOS system surface (a web sign-in
+ * sheet) arriving over the app, or leaving it (#2438). An app capture names none.
+ */
+function surfaceBundleChanged(baseline: SurfaceScopedNodes, snapshot: SnapshotState): boolean {
+ return baseline.surfaceBundleId !== snapshot.iosSystemSurfaceBundleId;
+}
+
+function sameProvenance(left: SnapshotState, right: SnapshotState): boolean {
+ return (
+ left.producer === right.producer &&
+ left.backend === right.backend &&
+ left.presentedSurface === right.presentedSurface &&
+ left.iosSystemSurfaceBundleId === right.iosSystemSurfaceBundleId
+ );
+}
diff --git a/packages/command-runtime/src/commands/interaction/runtime/settle.ts b/packages/command-runtime/src/commands/interaction/runtime/settle.ts
index dd0c64a38d..8aee0cb71c 100644
--- a/packages/command-runtime/src/commands/interaction/runtime/settle.ts
+++ b/packages/command-runtime/src/commands/interaction/runtime/settle.ts
@@ -92,7 +92,7 @@ export async function settleAfterInteraction(
): Promise {
return await settleAfterAction(runtime, options, {
...params,
- baseline: await resolveSettleBaseline(runtime, options, params.resolved),
+ ...(await resolveSettleBaseline(runtime, options, params.resolved, params.observationCurrent)),
actionPoint: params.resolved.point,
});
}
@@ -101,6 +101,11 @@ export type SettleObservationCommandOptions = CommandContext &
SettleParams & {
/** The pre-action tree the settled diff is taken against, and the surface it describes. */
baseline: SurfaceScopedNodes;
+ /**
+ * Whether `baseline` describes the screen right before this action. Absent means it may predate
+ * a side effect, so the adaptive confirmation never treats a change against it as the action's.
+ */
+ baselineFresh?: boolean;
};
/**
@@ -133,7 +138,11 @@ export const settleObservationCommand: RuntimeCommand<
async function settleAfterAction(
runtime: AgentDeviceRuntime,
options: CommandContext,
- params: SettleParams & { baseline: SurfaceScopedNodes; actionPoint?: Point },
+ params: SettleParams & {
+ baseline: SurfaceScopedNodes;
+ baselineFresh?: boolean;
+ actionPoint?: Point;
+ },
): Promise {
// The loop resolves an unrequested window from the capture that pays it, so the window is only
// known once one is taken. `base` carries the general default for the pre-capture shape the
@@ -154,6 +163,9 @@ async function settleAfterAction(
timeoutMs,
resetBudgetOnPrivateAxRecovery: true,
broadTransitionBaseline: params.baseline,
+ ...(params.earlyExit === 'adaptive'
+ ? { adaptiveConfirmation: { baselineFresh: params.baselineFresh === true } }
+ : {}),
});
if (outcome.failure) return settleUnavailable(base, outcome, outcome.failure.error);
return await readSettledOutcome(runtime, options, params, base, outcome);
@@ -237,9 +249,11 @@ async function resolveSettleBaseline(
runtime: AgentDeviceRuntime,
options: CommandContext,
resolved: ResolvedInteractionTarget,
-): Promise {
+ observationCurrent: boolean | undefined,
+): Promise<{ baseline: SurfaceScopedNodes; baselineFresh: boolean }> {
const session = await runtime.sessions.get(options.session ?? 'default');
const frame = surfaceScopedNodes(session?.refFrameSnapshot);
+ const refFrame = resolved.kind === 'ref' ? frame : undefined;
// A ref is authorized against the stored ref frame. Keep that visible presentation as the
// transition baseline: a best-effort evidence recapture can recover through private AX and see
// covered background controls that were not actionable when the ref was issued.
@@ -247,16 +261,22 @@ async function resolveSettleBaseline(
// authoritative ref frame, so reuse it rather than silently turning a missing optional field
// into an empty transition/diff baseline. Fall back to the latest observation for point targets
// and pre-frame sessions. A candidate read in another view than the frame is skipped (#4486).
- return postActionBaseline({
+ const baseline = await postActionBaseline({
frame: withoutPresentedSurface(frame),
candidates: [
- resolved.kind === 'ref' ? frame : undefined,
+ refFrame,
resolved.preAction,
session?.refFrameSnapshot
? undefined
: withoutPresentedSurface(surfaceScopedNodes(session?.snapshot)),
],
});
+ // Only a capture this action took (a selector resolution or a point's evidence read) proves the
+ // screen before it. A ref resolves against the stored frame, so its `preAction` is that frame too;
+ // the frame and the stored tree are fresh only while the daemon reports the observation current.
+ const actionCapture = resolved.kind !== 'ref' && baseline === resolved.preAction;
+ const baselineFresh = baseline.nodes.length > 0 && (actionCapture || observationCurrent === true);
+ return { baseline, baselineFresh };
}
/**
diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts
index fcc6a02c47..e2a37cd415 100644
--- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts
+++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture-signal.ts
@@ -53,6 +53,11 @@ export function stableCaptureSignalsHaveBroadReplacement(
// semantic replacement. Both signals carry the same chrome-free visible projection the settle
// diff collapses, and the transition baseline is only armed for XCTest sessions.
if (!left || left.nodes.length === 0 || right.nodes.length === 0) return false;
+ return sharedIdentityCount(left, right) / Math.max(left.nodes.length, right.nodes.length) < 0.5;
+}
+
+/** How many nodes the two signals share by semantic identity, each node matched at most once. */
+export function sharedIdentityCount(left: StableCaptureSignal, right: StableCaptureSignal): number {
const remainingIdentities = new Map();
for (const node of left.nodes) {
remainingIdentities.set(node.identity, (remainingIdentities.get(node.identity) ?? 0) + 1);
@@ -64,7 +69,7 @@ export function stableCaptureSignalsHaveBroadReplacement(
sharedNodes += 1;
remainingIdentities.set(node.identity, remaining - 1);
}
- return sharedNodes / Math.max(left.nodes.length, right.nodes.length) < 0.5;
+ return sharedNodes;
}
/**
diff --git a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts
index 0060264cd2..de7ed7665e 100644
--- a/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts
+++ b/packages/command-runtime/src/commands/interaction/runtime/stable-capture.ts
@@ -25,6 +25,7 @@ import {
} from './stable-capture-signal.ts';
import { preferredSnapshotBackendForVerdict } from '@agent-device/capture-kit/snapshot-quality-verdict';
import { withoutSettleChrome } from '@agent-device/capture-kit/snapshot-chrome';
+import type { AdaptiveEarlyExit } from './settle-adaptive-confirmation.ts';
/**
* The quiet-window stable-capture loop shared by `wait stable` and the
@@ -84,7 +85,8 @@ export type StableCaptureLoopResult = {
/**
* The quiet window the run was holding when it returned: the caller's, or the default resolved
* from the last capture that paid it. A run that completed no capture never priced a window, so
- * it reports the general default it started from.
+ * it reports the general default it started from. A run the adaptive early exit settled reports
+ * the span its confirming pair actually held instead.
*/
quietMs: number;
/** The most recent completed capture, settled or not. */
@@ -110,6 +112,11 @@ export async function runStableCaptureLoop(
* captures in its presentation, so the settled tree and the baseline share one view (#4486).
*/
broadTransitionBaseline?: SurfaceScopedNodes;
+ /**
+ * Internal prototype: settle on a baseline-differing capture that one short confirmation agrees
+ * with (see settle-adaptive-confirmation.ts). Absent keeps the quiet window alone.
+ */
+ adaptiveConfirmation?: { baselineFresh: boolean };
},
): Promise {
const { timeoutMs } = params;
@@ -135,6 +142,7 @@ export async function runStableCaptureLoop(
// loop recovers on private AX, the rest of the loop stays on it.
let preferredBackend: SnapshotPreferredBackend | undefined;
const pacing = createStableCapturePacing(start);
+ const adaptive = await loadAdaptiveEarlyExit(params, session?.appBundleId);
let captures = 0;
let lastCapture: CapturedSnapshot | undefined;
const result = (
@@ -159,14 +167,7 @@ export async function runStableCaptureLoop(
runtime,
options,
deadlineMs - now(runtime),
- params.broadTransitionBaseline
- ? postActionPresentation(params.broadTransitionBaseline)
- : {
- ...postActionPresentation(undefined),
- depth: options.depth,
- scope: options.scope,
- raw: options.raw,
- },
+ stableCapturePresentation(params.broadTransitionBaseline, options),
preferredBackend,
);
if (read.kind !== 'captured') return unfinished(read);
@@ -199,17 +200,110 @@ export async function runStableCaptureLoop(
if (recovered) {
privateAxRecoveryBudgetReset = true;
deadlineMs = recovered.deadlineMs;
+ adaptive?.disarm();
}
const step = recovered
? { settled: false, delayMs: recovered.delayMs }
- : pacing.observe({ signal, nodeCount, nowMs, requiredQuietMs, pollMs, deadlineMs, captures });
- if (step.settled) return result(true, false, nowMs - start);
+ : observeStableCapture({
+ pacing,
+ adaptive,
+ confirmation,
+ snapshot: capture.snapshot,
+ signal,
+ window,
+ nowMs,
+ deadlineMs,
+ captures,
+ });
+ if (step.settled) {
+ quietMs = step.heldQuietMs ?? quietMs;
+ return result(true, false, nowMs - start);
+ }
if (step.delayMs <= 0) break;
await sleep(runtime, step.delayMs);
}
return result(false, false, now(runtime) - start);
}
+/**
+ * The view every capture of the run reads: the baseline's, so the settled tree and the baseline
+ * compare (#4486), or without one the interactive view the caller's options narrow.
+ */
+function stableCapturePresentation(
+ baseline: SurfaceScopedNodes | undefined,
+ options: SelectorSnapshotOptions,
+): SnapshotPresentation {
+ if (baseline) return postActionPresentation(baseline);
+ return {
+ ...postActionPresentation(undefined),
+ depth: options.depth,
+ scope: options.scope,
+ raw: options.raw,
+ };
+}
+
+/**
+ * The internal adaptive early exit, loaded only when a settle asks for it so the default path keeps
+ * its import closure.
+ */
+async function loadAdaptiveEarlyExit(
+ params: {
+ broadTransitionBaseline?: SurfaceScopedNodes;
+ adaptiveConfirmation?: { baselineFresh: boolean };
+ },
+ appBundleId: string | undefined,
+): Promise {
+ if (!params.adaptiveConfirmation) return undefined;
+ const { createAdaptiveConfirmation } = await import('./settle-adaptive-confirmation.ts');
+ return createAdaptiveConfirmation({
+ baseline: params.broadTransitionBaseline,
+ baselineFresh: params.adaptiveConfirmation.baselineFresh,
+ appBundleId,
+ });
+}
+
+/**
+ * Folds one capture into the quiet window and, when the adaptive early exit is armed, into its
+ * short confirmation: settled, or how long to wait before the next capture (<= 0 ends the run).
+ */
+function observeStableCapture(params: {
+ pacing: StableCapturePacing;
+ adaptive: AdaptiveEarlyExit | undefined;
+ confirmation: StableCaptureConfirmation;
+ snapshot: SnapshotState;
+ signal: StableCaptureSignal;
+ window: ReturnType;
+ nowMs: number;
+ deadlineMs: number;
+ captures: number;
+}): { settled: boolean; delayMs: number; heldQuietMs?: number } {
+ const { window, adaptive } = params;
+ const step = params.pacing.observe({
+ signal: params.signal,
+ nodeCount: params.snapshot.nodes.length,
+ nowMs: params.nowMs,
+ requiredQuietMs: window.requiredQuietMs,
+ pollMs: window.pollMs,
+ deadlineMs: params.deadlineMs,
+ captures: params.captures,
+ });
+ if (step.settled || !adaptive) return step;
+ const early = adaptive.observe({
+ snapshot: params.snapshot,
+ signal: params.signal,
+ nowMs: params.nowMs,
+ floorInForce:
+ window.floorConfirmed || params.confirmation.unresolvedReplacement(params.snapshot),
+ });
+ if (early.kind === 'settle') {
+ return { settled: true, delayMs: 0, heldQuietMs: early.heldQuietMs };
+ }
+ if (early.kind === 'confirm' && step.delayMs > 0) {
+ return { settled: false, delayMs: Math.min(step.delayMs, adaptive.confirmGapMs) };
+ }
+ return step;
+}
+
/**
* The quiet window one capture prices, the window it must actually hold once the broad-transition
* confirmation has had its say, and the cadence that can measure that. Sampling an extended wait on
@@ -219,10 +313,15 @@ function stableCaptureWindow(
snapshot: SnapshotState,
requestedQuietMs: number | undefined,
confirmation: StableCaptureConfirmation,
-): { quietMs: number; requiredQuietMs: number; pollMs: number } {
+): { quietMs: number; requiredQuietMs: number; pollMs: number; floorConfirmed: boolean } {
const quietMs = requestedQuietMs ?? defaultQuietMsForCapture(snapshot);
const requiredQuietMs = confirmation.requiredQuietMs(snapshot, quietMs);
- return { quietMs, requiredQuietMs, pollMs: stableCapturePollMs(requiredQuietMs) };
+ return {
+ quietMs,
+ requiredQuietMs,
+ pollMs: stableCapturePollMs(requiredQuietMs),
+ floorConfirmed: confirmation.confirmed,
+ };
}
/**
@@ -404,6 +503,19 @@ function broadTransitionConfirmation(params: {
? currentQuietMs
: Math.max(currentQuietMs, confirmedQuietMs);
},
+ get confirmed() {
+ return confirmedQuietMs !== undefined;
+ },
+ unresolvedReplacement(snapshot) {
+ // The gate has refused every capture so far, so the latch has not ruled; a later capture it
+ // admits could still confirm the floor. Judge this capture against the baseline directly.
+ if (!baseline || transitionBaseline !== undefined) return false;
+ if (!isBoundIosXCTestCapture(snapshot.backend, platform)) return false;
+ return stableCaptureSignalsHaveBroadReplacement(
+ stableCaptureComparisonSignal(snapshot, baseline.nodes, appBundleId),
+ stableCaptureComparisonSignal(snapshot, snapshot.nodes, appBundleId),
+ );
+ },
};
}
@@ -413,6 +525,17 @@ type StableCaptureConfirmation = {
snapshot: Parameters[0],
currentQuietMs: number,
) => number;
+ /**
+ * A capture proved the broad replacement, so the floor holds for the rest of the run, whether or
+ * not it raised the window: a requested window above the floor still owes its full span.
+ */
+ readonly confirmed: boolean;
+ /**
+ * The latch is armed but no capture has passed its gate yet (a rootless read after a
+ * non-presented baseline), and this capture already replaces the baseline broadly. Asked only by
+ * the adaptive early exit, which must not settle before a later capture could confirm the floor.
+ */
+ unresolvedReplacement: (snapshot: Parameters[0]) => boolean;
};
/**
diff --git a/packages/contracts/src/daemon-command-surface.ts b/packages/contracts/src/daemon-command-surface.ts
index 85e86560bc..03c4c1cfe2 100644
--- a/packages/contracts/src/daemon-command-surface.ts
+++ b/packages/contracts/src/daemon-command-surface.ts
@@ -300,6 +300,8 @@ export type DaemonSettleObservationOptions = CommandContext &
Readonly<{
/** The pre-action tree the settled diff is taken against, and the surface it describes. */
baseline: SurfaceScopedNodes;
+ /** Whether `baseline` describes the screen right before this action (adaptive settle only). */
+ baselineFresh?: boolean;
}>;
/**
diff --git a/packages/contracts/src/interaction.ts b/packages/contracts/src/interaction.ts
index edd81f0787..97d6198c4d 100644
--- a/packages/contracts/src/interaction.ts
+++ b/packages/contracts/src/interaction.ts
@@ -301,6 +301,18 @@ export type SettleTailEntry = {
export type SettleParams = {
quietMs?: number;
timeoutMs?: number;
+ /**
+ * Internal prototype switch, never a CLI flag: 'adaptive' lets a baseline-differing capture that one
+ * short confirmation agrees with settle before the quiet window. The daemon sets it only when the
+ * host environment opts in (AGENT_DEVICE_SETTLE_ADAPTIVE=1).
+ */
+ earlyExit?: 'adaptive';
+ /**
+ * With `earlyExit`: when the action began, the session's ref frame was active, its tree was the
+ * newest observation, and no native read had outdated it. A baseline this action did not capture
+ * itself (the ref frame or the stored tree) is fresh only then.
+ */
+ observationCurrent?: boolean;
};
/**
@@ -319,6 +331,10 @@ export type SettleObservation = {
settled: boolean;
waitedMs: number;
captures: number;
+ /**
+ * The quiet window the settle held: the requested or producer default window, or, when the
+ * internal adaptive early exit settled the run, the span its confirming pair actually held.
+ */
quietMs: number;
timeoutMs: number;
/**
diff --git a/packages/daemon/src/__tests__/support/store-factory.ts b/packages/daemon/src/__tests__/support/store-factory.ts
index cf2efaa757..5ea996ebb8 100644
--- a/packages/daemon/src/__tests__/support/store-factory.ts
+++ b/packages/daemon/src/__tests__/support/store-factory.ts
@@ -1,6 +1,10 @@
import path from 'node:path';
import { SessionStore } from '../../core/session-store.ts';
-import { createLiveTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit';
+import {
+ createLiveTestHostEnvironment,
+ createTestHostEnvironment,
+ mkdtempForTestSync,
+} from '@agent-device/test-kit';
import type { SessionRef, SessionState } from '../../core/session-state.ts';
export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore {
@@ -8,6 +12,16 @@ export function makeSessionStore(prefix = 'agent-device-test-'): SessionStore {
return new SessionStore(path.join(tempRoot, 'sessions'), createLiveTestHostEnvironment());
}
+/** A store whose daemon host environment adds `env`, such as an internal opt-in switch. */
+export function makeSessionStoreWithEnv(env: Record): SessionStore {
+ const tempRoot = mkdtempForTestSync('agent-device-test-');
+ const host = createTestHostEnvironment();
+ return new SessionStore(path.join(tempRoot, 'sessions'), {
+ ...host,
+ env: { ...host.env, ...env },
+ });
+}
+
export function makeStoredSessionRef(session: SessionState, address = session.name): SessionRef {
return makeSessionStore().publish(address, session);
}
diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts
index 1ef7baa1ca..807fc0dbb4 100644
--- a/packages/daemon/src/core/deferred-interaction-outcome.ts
+++ b/packages/daemon/src/core/deferred-interaction-outcome.ts
@@ -3,7 +3,7 @@ import type { SettleObservation, SurfaceScopedNodes } from '@agent-device/contra
import type { PostOpenObservation } from '@agent-device/contracts/application-lifecycle-runtime';
import type { SnapshotCaptureAnnotations } from '@agent-device/contracts/capture';
import { isApplePlatform, isMobilePlatform } from '@agent-device/kernel/device';
-import { formatRole, type SnapshotNode } from '@agent-device/kernel/snapshot';
+import { formatRole, isBusyIndicatorNode, type SnapshotNode } from '@agent-device/kernel/snapshot';
import type { SnapshotState } from '@agent-device/contracts/snapshot-model';
import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract';
import {
@@ -290,7 +290,7 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean {
const reading = node.label?.trim() || node.value?.trim();
if (!reading) continue;
const role = formatRole(node.type ?? '');
- if (BUSY_INDICATOR_ROLES.has(role)) continue;
+ if (isBusyIndicatorNode(node)) continue;
if (role !== 'text') return true;
texts.add(reading);
if (texts.size > 1) return true;
@@ -298,11 +298,6 @@ function showsFirstScreen(nodes: readonly SnapshotNode[]): boolean {
return false;
}
-const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([
- 'activity-indicator',
- 'progress-indicator',
-]);
-
function clearSurfaceStabilization(session: SessionState | undefined): void {
if (session?.deferredSurface?.kind !== 'stabilize') return;
session.deferredSurface = undefined;
diff --git a/packages/daemon/src/execution/generic-settle.test.ts b/packages/daemon/src/execution/generic-settle.test.ts
index f301e83e36..37ad70a295 100644
--- a/packages/daemon/src/execution/generic-settle.test.ts
+++ b/packages/daemon/src/execution/generic-settle.test.ts
@@ -7,7 +7,7 @@ import type { CommandFlags } from '@agent-device/contracts/command';
import type { PresentedSurface, RawSnapshotNode } from '@agent-device/kernel/snapshot';
import type { IosSystemSurfaceProvenance } from '@agent-device/contracts/ios-system-surface';
import { makeIosSession } from '../__tests__/support/session-factories.ts';
-import { makeSessionStore } from '../__tests__/support/store-factory.ts';
+import { makeSessionStore, makeSessionStoreWithEnv } from '../__tests__/support/store-factory.ts';
import {
activateCompleteRefFrame,
markSessionSnapshotOutdated,
@@ -712,3 +712,75 @@ for (const change of ['rebuild', 'retire'] as const) {
}
});
}
+
+const ADAPTIVE_ENV = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' };
+
+test('back --settle with the adaptive switch arms a current stored tree as a fresh baseline', async () => {
+ const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV);
+ const sessionName = 'generic-settle-adaptive-current';
+ seedSession(sessionName, sessionStore);
+ mockCommandDispatch([AFTER_NODES]);
+
+ await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } });
+
+ expect(settleObservationCalls[0]).toMatchObject({
+ quietMs: 25,
+ timeoutMs: 2_000,
+ earlyExit: 'adaptive',
+ observationCurrent: true,
+ baselineFresh: true,
+ });
+});
+
+test('back --settle with the adaptive switch does not trust a stored tree a native read outdated', async () => {
+ const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV);
+ const sessionName = 'generic-settle-adaptive-outdated';
+ const session = seedSession(sessionName, sessionStore);
+ markSessionSnapshotOutdated(session);
+ mockCommandDispatch([AFTER_NODES]);
+
+ await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } });
+
+ expect(settleObservationCalls[0]).toMatchObject({
+ earlyExit: 'adaptive',
+ observationCurrent: false,
+ baselineFresh: false,
+ });
+});
+
+test('back --settle with the adaptive switch does not trust a baseline the frame answered', async () => {
+ const sessionStore = makeSessionStoreWithEnv(ADAPTIVE_ENV);
+ const sessionName = 'generic-settle-adaptive-frame';
+ // The frame was read interactive-only; a later full read replaced the stored tree, so the frame
+ // answers the baseline and a newer observation exists.
+ const session = makeIosSession(sessionName);
+ setSessionSnapshot(
+ session,
+ buildSnapshotState(snapshotPayload({ nodes: BEFORE_NODES }), { snapshotInteractiveOnly: true }),
+ );
+ activateCompleteRefFrame(session);
+ setSessionSnapshot(session, buildSnapshotState(snapshotPayload({ nodes: AFTER_NODES }), {}));
+ sessionStore.publish(sessionName, session);
+ mockCommandDispatch([AFTER_NODES]);
+
+ await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } });
+
+ expect(settleObservationCalls[0]?.baseline.presentation?.interactiveOnly).toBe(true);
+ expect(settleObservationCalls[0]).toMatchObject({
+ observationCurrent: false,
+ baselineFresh: false,
+ });
+});
+
+test('back --settle without the adaptive switch forwards no early exit', async () => {
+ const sessionStore = makeSessionStore();
+ const sessionName = 'generic-settle-adaptive-off';
+ seedSession(sessionName, sessionStore);
+ mockCommandDispatch([AFTER_NODES]);
+
+ await dispatchGeneric({ sessionName, sessionStore, command: 'back', flags: { ...SETTLE_FLAGS } });
+
+ expect(settleObservationCalls[0]?.earlyExit).toBeUndefined();
+ expect(settleObservationCalls[0]?.observationCurrent).toBeUndefined();
+ expect(settleObservationCalls[0]?.baselineFresh).toBe(false);
+});
diff --git a/packages/daemon/src/execution/generic-settle.ts b/packages/daemon/src/execution/generic-settle.ts
index f2accd1537..3d9cfd889c 100644
--- a/packages/daemon/src/execution/generic-settle.ts
+++ b/packages/daemon/src/execution/generic-settle.ts
@@ -90,16 +90,22 @@ export async function planGenericSettleObservation(
if (!commandSupportsSettleObservation(params.command)) return {};
const invalidSettleFlags = settleFlagGuardResponse(params.command, params.flags);
if (invalidSettleFlags) return { response: invalidSettleFlags };
- const settle = readSettleRequest(params.flags);
+ const settle = readSettleRequest(params.flags, {
+ env: params.sessionStore.host.env,
+ session: params.session,
+ });
if (!settle) return {};
const { presentedSurface: _unverified, ...frame } = surfaceScopedNodes(
refFrameTree(params.session),
);
- const baseline = await postActionBaseline({
- frame,
- candidates: [storedTreeBaseline(params.session.snapshot)],
- });
- return { observe: async () => await observeSettled(params, settle, baseline) };
+ const stored = storedTreeBaseline(params.session.snapshot);
+ const baseline = await postActionBaseline({ frame, candidates: [stored] });
+ // The generic route captures nothing before the action, so the stored tree is fresh only while the
+ // daemon reports the session observation current.
+ const baselineFresh = baseline === stored && settle.observationCurrent === true;
+ return {
+ observe: async () => await observeSettled(params, { ...settle, baselineFresh }, baseline),
+ };
}
/**
@@ -124,7 +130,7 @@ function storedTreeBaseline(snapshot: SnapshotState | undefined): SurfaceScopedN
*/
async function observeSettled(
context: GenericSettleContext,
- settle: SettleParams,
+ settle: SettleParams & { baselineFresh: boolean },
baseline: SurfaceScopedNodes,
): Promise {
const runtime = await createGenericSettleRuntime(context);
diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts
new file mode 100644
index 0000000000..4b50808476
--- /dev/null
+++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.test.ts
@@ -0,0 +1,78 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import { makeSession } from '../../../__tests__/support/session-factories.ts';
+import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures';
+import {
+ activateCompleteRefFrame,
+ expireRefFrame,
+ markSessionSnapshotOutdated,
+} from '../../../core/ref-frame.ts';
+import { readSettleRequest } from './interaction-flags.ts';
+
+const OPT_IN = { AGENT_DEVICE_SETTLE_ADAPTIVE: '1' };
+
+/** A session whose stored tree was published as a complete ref frame (`snapshot`). */
+function sessionWithFrame() {
+ const session = makeSession('default', {
+ snapshot: makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Next' }]),
+ });
+ activateCompleteRefFrame(session);
+ return session;
+}
+
+test('the host environment opts a settle request into the adaptive confirmation', () => {
+ const session = sessionWithFrame();
+
+ assert.deepEqual(
+ readSettleRequest(
+ { settle: true, settleQuietMs: 700, timeoutMs: 4_000 },
+ { env: OPT_IN, session },
+ ),
+ { quietMs: 700, timeoutMs: 4_000, earlyExit: 'adaptive', observationCurrent: true },
+ );
+});
+
+test('a ref frame a side effect crossed is not current for the adaptive confirmation', () => {
+ const session = sessionWithFrame();
+ expireRefFrame(session);
+
+ assert.equal(
+ readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent,
+ false,
+ );
+});
+
+test('a frame whose tree a native read outdated is not current for the adaptive confirmation', () => {
+ // `wait text` reads the device natively and marks the stored tree outdated without expiring the
+ // frame.
+ const session = sessionWithFrame();
+ markSessionSnapshotOutdated(session);
+
+ assert.equal(
+ readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent,
+ false,
+ );
+});
+
+test('a newer observation than the frame is not current for the adaptive confirmation', () => {
+ // An Android freshness capture stores a newer tree while the frame stays active.
+ const session = sessionWithFrame();
+ session.snapshot = makeSnapshotState([{ index: 0, depth: 0, type: 'Button', label: 'Done' }]);
+
+ assert.equal(
+ readSettleRequest({ settle: true }, { env: OPT_IN, session })?.observationCurrent,
+ false,
+ );
+});
+
+test('a settle request without the opt-in keeps the quiet window rule', () => {
+ const session = makeSession('default');
+ const env = { AGENT_DEVICE_SETTLE_ADAPTIVE: 'true' };
+
+ assert.deepEqual(readSettleRequest({ settle: true }, { env, session }), {});
+ assert.deepEqual(readSettleRequest({ settle: true }), {});
+});
+
+test('the opt-in alone requests no settle', () => {
+ assert.equal(readSettleRequest({}, { env: OPT_IN, session: makeSession('default') }), undefined);
+});
diff --git a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts
index 45abb96482..b0ea3db2fa 100644
--- a/packages/daemon/src/execution/interaction/internal/interaction-flags.ts
+++ b/packages/daemon/src/execution/interaction/internal/interaction-flags.ts
@@ -1,5 +1,8 @@
import type { DaemonResponse } from '@agent-device/kernel/contracts';
import type { CommandFlags } from '@agent-device/contracts/command';
+import type { EnvMap } from '@agent-device/kernel/source-value';
+import { isOutdatedObservation, refFrameState, refFrameTree } from '../../../core/ref-frame.ts';
+import type { SessionState } from '../../../core/session-state.ts';
import type { SettleParams } from '@agent-device/contracts/interaction';
import { interactionErrorResponse } from './interaction-response.ts';
@@ -17,10 +20,43 @@ export function settleFlagGuardResponse(
);
}
-export function readSettleRequest(flags: CommandFlags | undefined): SettleParams | undefined {
+const ADAPTIVE_SETTLE_ENV = 'AGENT_DEVICE_SETTLE_ADAPTIVE';
+
+/**
+ * The session's observation still describes the screen: the ref frame is active (no side-effect
+ * seam expired it), no native read marked its tree outdated (`wait text`), and no newer
+ * observation replaced it (an Android freshness capture).
+ */
+function isObservationCurrent(session: SessionState): boolean {
+ const tree = refFrameTree(session);
+ return (
+ refFrameState(session) === 'active' &&
+ tree !== undefined &&
+ session.snapshot === tree &&
+ !isOutdatedObservation(tree)
+ );
+}
+
+/**
+ * The settle request a command carries. `host` selects the internal adaptive confirmation prototype
+ * through the daemon host environment and nothing else. Read it before the action dispatches: the
+ * dispatch expires the ref frame, and the prototype asks whether the observation was current before
+ * it.
+ */
+export function readSettleRequest(
+ flags: CommandFlags | undefined,
+ host?: { env: Readonly; session: SessionState | undefined },
+): SettleParams | undefined {
if (flags?.settle !== true) return undefined;
+ const adaptive = host?.env[ADAPTIVE_SETTLE_ENV] === '1';
return {
...(flags.settleQuietMs !== undefined ? { quietMs: flags.settleQuietMs } : {}),
...(flags.timeoutMs !== undefined ? { timeoutMs: flags.timeoutMs } : {}),
+ ...(adaptive
+ ? {
+ earlyExit: 'adaptive' as const,
+ observationCurrent: host.session !== undefined && isObservationCurrent(host.session),
+ }
+ : {}),
};
}
diff --git a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts
index 0cd73501b6..bc1efd50c3 100644
--- a/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts
+++ b/packages/daemon/src/execution/interaction/internal/interaction-settle.test.ts
@@ -12,7 +12,10 @@ import {
refFrameState,
refFrameTree,
} from '../../../core/ref-frame.ts';
-import { makeSessionStore } from '../../../__tests__/support/store-factory.ts';
+import {
+ makeSessionStore,
+ makeSessionStoreWithEnv,
+} from '../../../__tests__/support/store-factory.ts';
import { makeIosSession } from '../../../__tests__/support/session-factories.ts';
import {
clearRequestAbortRegistration,
@@ -682,3 +685,51 @@ test('fill @ref --settle carries the settle payload on the ref wire shape', asyn
expect(settle.settled).toBe(true);
expect(typeof settle.refsGeneration).toBe('number');
});
+
+// hover is web-only (refused on this iOS session); it reads the same shared settle options.
+for (const command of ['press', 'click', 'longpress'] as const) {
+ for (const adaptive of [true, false]) {
+ test(`${command} --settle forwards ${adaptive ? 'the adaptive early exit' : 'only its window'} to the settle observation`, async () => {
+ const sessionStore = adaptive
+ ? makeSessionStoreWithEnv({ AGENT_DEVICE_SETTLE_ADAPTIVE: '1' })
+ : makeSessionStore();
+ const sessionName = `settle-forward-${command}-${adaptive}`;
+ seedSession(sessionName, sessionStore);
+ mockCommandDispatch({ snapshots: [BEFORE_NODES, AFTER_NODES] });
+ const forwarded: unknown[] = [];
+ const recordingSurface = createTouchCommandSurface({
+ observeSettle: async ({ settle }) => {
+ forwarded.push(settle);
+ return { settled: true, waitedMs: 0, captures: 1, quietMs: 25, timeoutMs: 2_000 };
+ },
+ });
+
+ const response = await handleInteractionCommands({
+ createCommandSurface: recordingSurface,
+ req: {
+ token: 't',
+ session: sessionName,
+ command,
+ positionals: ['label=Continue'],
+ flags: { ...SETTLE_FLAGS },
+ },
+ sessionName,
+ sessionStore,
+ contextFromFlags,
+ captureSnapshotForSession: mockCaptureSnapshotForSession,
+ ...getRuntimeBindings(),
+ });
+
+ expectOkData(response);
+ // The seeded snapshot published a complete frame and nothing outdated it, so the observation
+ // is current before the action.
+ expect(forwarded).toEqual([
+ {
+ quietMs: 25,
+ timeoutMs: 2_000,
+ ...(adaptive ? { earlyExit: 'adaptive', observationCurrent: true } : {}),
+ },
+ ]);
+ });
+ }
+}
diff --git a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts
index a5e114df65..d8e300d9a2 100644
--- a/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts
+++ b/packages/daemon/src/execution/interaction/internal/interaction-touch-fill.ts
@@ -78,7 +78,10 @@ export async function dispatchFillViaRuntime(params: FillParams): Promise;
flags: CommandFlags | undefined;
+ settle: ReturnType;
durationMs?: number;
expectedResolvedTarget?: ReplayTargetGuardDenotation;
/** #1654: a mutating `find`'s already-resolved node; see daemon/daemon-request.ts. */
preresolvedTarget?: PreresolvedInteractionTarget;
}): Promise {
- const { runtime, command, target, sessionName, requestId, flags, expectedResolvedTarget } =
- params;
+ const { runtime, command, target, sessionName, requestId, expectedResolvedTarget } = params;
const shared = {
session: sessionName,
requestId,
- settle: readSettleRequest(flags),
+ settle: params.settle,
expectedResolvedTarget,
};
switch (command) {
diff --git a/packages/kernel/src/snapshot.ts b/packages/kernel/src/snapshot.ts
index e8dc06b0c0..d36aded223 100644
--- a/packages/kernel/src/snapshot.ts
+++ b/packages/kernel/src/snapshot.ts
@@ -573,6 +573,20 @@ export function formatRole(type: string): string {
return lookupRoleLabel(normalized) || normalized || 'element';
}
+const BUSY_INDICATOR_ROLES: ReadonlySet = new Set([
+ 'activity-indicator',
+ 'progress-indicator',
+ 'progressbar',
+]);
+
+/**
+ * Whether a node is a busy indicator: a spinner or progress element that says only that the screen
+ * is not ready yet. Apple reports activity and progress indicators; Android reports a ProgressBar.
+ */
+export function isBusyIndicatorNode(node: Pick): boolean {
+ return BUSY_INDICATOR_ROLES.has(formatRole(node.type ?? ''));
+}
+
/**
* Reasons the Apple runner can stamp when serving a command required re-activating the session app
* (#2682). Mirrors its `activateTarget(bundleId:reason:)` call sites.