diff --git a/android/snapshot-helper/README.md b/android/snapshot-helper/README.md index c7bc9ca187..39c0b3df70 100644 --- a/android/snapshot-helper/README.md +++ b/android/snapshot-helper/README.md @@ -184,6 +184,6 @@ resolve the APK from `apkUrl`, verify `sha256`, install it with `adb install -r `instrumentationRunner`. Manifests up to 0.21.3 also carried an `installArgs` array; it only ever spelled `install -r` (plus `-t` while the helper was `testOnly`), so consumers can ignore it. -Capture metadata reports `cacheInvalidated` when the capture cleared the platform node cache through `UiAutomation.clearCache()`. A complete capture with that evidence need not change routes after a control press. Incomplete captures and captures without that evidence retain freshness recovery. +Capture metadata reports `cacheInvalidated` when the capture cleared the platform node cache through `UiAutomation.clearCache()`. A complete capture with that evidence need not change routes after a control press. A press or click opens no freshness window when its `--settle` converged, stored its final capture as the session snapshot, and that capture is complete with that evidence, so the next command reads without freshness recovery. Incomplete captures and captures without that evidence retain freshness recovery. Warm application open commands use the same automation session. Close retains the existing foreground and process-absence checks. A failed application command is never resent. diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts index b858598edb..b843ace5e4 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/android.ts @@ -1,3 +1,4 @@ +import type { AndroidSnapshotBackendMetadata } from '@agent-device/contracts/capture'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { SnapshotFreshnessAttempt, @@ -19,6 +20,22 @@ export const ANDROID_COMPARISON_BASELINE_MAX_AGE_MS = 5_000; export const ANDROID_FRESHNESS_RETRY_BUDGET_MS = 1_500; export const ANDROID_FRESHNESS_RETRY_DELAYS_MS = [250, 400, 600] as const; +/** + * Whether the helper proved this capture a complete read of the current hierarchy: it cleared the + * platform node cache first, read a root, truncated nothing, and serialized every listed window + * root. Captures without this proof can agree with each other on a stale tree. + */ +export function provesFreshAndroidTraversal( + metadata: AndroidSnapshotBackendMetadata | undefined, +): boolean { + return ( + metadata?.cacheInvalidated === true && + metadata.rootPresent === true && + metadata.helperTruncated === false && + metadata.missingRootWindowTypes?.length === 0 + ); +} + export function isNavigationSensitiveAction(command: string): boolean { // Keep this set intentionally narrow. `type`, `fill`, and generic `swipe` happen far more // often than real route changes, so marking freshness for them would add retry latency to diff --git a/packages/capture-kit/src/snapshot/snapshot-freshness/index.ts b/packages/capture-kit/src/snapshot/snapshot-freshness/index.ts index 63a1e04915..594d2d0efc 100644 --- a/packages/capture-kit/src/snapshot/snapshot-freshness/index.ts +++ b/packages/capture-kit/src/snapshot/snapshot-freshness/index.ts @@ -7,6 +7,7 @@ export { buildSnapshotSignatures, isLikelyStaleSnapshotDrop, isNavigationSensitiveAction, + provesFreshAndroidTraversal, } from './android.ts'; export { captureFreshnessRecoveredAttempt, diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index fd0b8cad97..586afe579c 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -186,7 +186,7 @@ const SHARED_RESPONSE_CONSTRUCTION: GuaranteeEnforcement = { const TAP_OUTCOME_NOT_OBSERVED_GAP: GuaranteeEnforcement = { kind: 'waived', reason: - 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization), unless the request sets postGestureStabilization false; that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', + 'gap: the response reports the dispatch only; only opt-in --verify/--settle capture post-action evidence into it. The deferred marks set after dispatch (Android snapshot freshness after press/click unless its --settle converged on a capture the helper proved fresh, post-gesture stabilization when the request sets postGestureStabilization) are judged by the next capture, never in this response. An iOS open that launched an app (any postOpenObservation) marks the next capture in that session to wait for the first screen of that app (markPostOpenStabilization), unless the request sets postGestureStabilization false; that steadies where a target is resolved, not whether the tap landed. The iOS ambiguous-failure corroboration reconsiders only a thrown runner error.', trackingIssue: TAP_OUTCOME_NOT_OBSERVED_GAP_ISSUE, }; diff --git a/packages/contracts/src/snapshot-model.ts b/packages/contracts/src/snapshot-model.ts index e472018cca..3928522e59 100644 --- a/packages/contracts/src/snapshot-model.ts +++ b/packages/contracts/src/snapshot-model.ts @@ -105,6 +105,11 @@ export type SnapshotState = { * occlusion (see `@agent-device/contracts/android-system-surface-disclosure`). */ systemSurfaceOnly?: boolean; + /** + * Android: the producer proved this capture a complete read of the current hierarchy, taken after + * it cleared the platform node cache. Absent proves nothing. + */ + freshTraversal?: true; /** * iOS: the bundle id of the in-place system surface this capture describes (a web sign-in sheet * presented over the app, #2438). Two captures that disagree here describe different surfaces and diff --git a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts index 8598b4d1ba..73003b2cfc 100644 --- a/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts +++ b/packages/daemon/src/core/__tests__/deferred-interaction-outcome.test.ts @@ -11,6 +11,7 @@ import { } from '../deferred-interaction-outcome.ts'; import type { SessionState } from '../session-state.ts'; import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; +import type { SettleObservation } from '@agent-device/contracts/interaction'; import { makeSnapshotState } from '@agent-device/selectors/snapshot-geometry-fixtures'; import { deliverySnapshot, @@ -89,6 +90,58 @@ test('marking is one call for both flags, each keeping its own eligibility gate' assert.equal(isPostGestureStabilizationPending(session), false); }); +const convergedSettle: SettleObservation = { + settled: true, + waitedMs: 600, + captures: 2, + quietMs: 300, + timeoutMs: 3000, + diff: { summary: { additions: 1, removals: 1, unchanged: 4 }, lines: [] }, +}; + +test('a press whose settle stored a proved-fresh tree retires an earlier freshness window', () => { + const session = makeSession('android'); + markDeferredInteractionOutcome({ session, command: 'press', positionals: [], flags: undefined }); + session.snapshot = { ...pickupSnapshot(), freshTraversal: true }; + + markDeferredInteractionOutcome({ + session, + command: 'press', + positionals: [], + flags: undefined, + settle: convergedSettle, + }); + + assert.equal(session.androidSnapshotFreshness, undefined); +}); + +test('a press whose settle never converged keeps the freshness window over a fresh read', () => { + const { diff: _diff, ...unconverged } = convergedSettle; + const session = makeSession('android'); + session.snapshot = { ...pickupSnapshot(), freshTraversal: true }; + + markDeferredInteractionOutcome({ + session, + command: 'press', + positionals: [], + flags: undefined, + settle: { ...unconverged, settled: false }, + }); + + assert.equal(session.androidSnapshotFreshness?.action, 'press'); +}); + +// #4105: the generic route marks a bare `back` with no settle observation. A proved-fresh tree +// stored before the back is no proof about the screen the back navigated to. +test('a bare back keeps the freshness window over a fresh stored tree', () => { + const session = makeSession('android'); + session.snapshot = { ...pickupSnapshot(), freshTraversal: true }; + + markDeferredInteractionOutcome({ session, command: 'back', positionals: [], flags: undefined }); + + assert.equal(session.androidSnapshotFreshness?.action, 'back'); +}); + test('a scroll marks stabilization but not freshness', () => { const session = makeSession('android'); session.snapshot = pickupSnapshot(); diff --git a/packages/daemon/src/core/deferred-interaction-outcome.ts b/packages/daemon/src/core/deferred-interaction-outcome.ts index 09cda53f68..1ef7baa1ca 100644 --- a/packages/daemon/src/core/deferred-interaction-outcome.ts +++ b/packages/daemon/src/core/deferred-interaction-outcome.ts @@ -8,6 +8,7 @@ import type { SnapshotState } from '@agent-device/contracts/snapshot-model'; import type { MovingTargetBaseline } from '@agent-device/contracts/runtime-contract'; import { captureAndroidFreshnessRecoveredAttempt, + clearAndroidSnapshotFreshness, getActiveAndroidSnapshotFreshness, markAndroidSnapshotFreshness, } from './session-snapshot-freshness.ts'; @@ -88,7 +89,15 @@ export function markDeferredInteractionOutcome( settle, } = params; if (isNavigationSensitiveAction(action)) { - markAndroidSnapshotFreshness(session, action, androidFreshnessBaseline ?? session.snapshot); + // A converged settle stored its final capture as the session snapshot (`diff` is attached only + // then). When the helper also proved that capture a complete read after a cache reset, it is + // already the fresh post-action tree the window would wait for. Convergence alone is no proof: + // two stale dumps can agree. + if (settle?.diff !== undefined && session.snapshot?.freshTraversal === true) { + clearAndroidSnapshotFreshness(session); + } else { + markAndroidSnapshotFreshness(session, action, androidFreshnessBaseline ?? session.snapshot); + } } // The two post-action mechanisms are complementary, and this is the seam that keeps them apart. // A gesture-shaped mutation hands its unsettled surface to the capture side: every following diff --git a/packages/daemon/src/core/session-snapshot-freshness.ts b/packages/daemon/src/core/session-snapshot-freshness.ts index 8e8e3495c1..cf32fff067 100644 --- a/packages/daemon/src/core/session-snapshot-freshness.ts +++ b/packages/daemon/src/core/session-snapshot-freshness.ts @@ -7,6 +7,7 @@ import { androidFreshnessReason, buildSnapshotSignatures, captureFreshnessRecoveredAttempt, + provesFreshAndroidTraversal, type SnapshotFreshnessAttemptShape, type SnapshotFreshnessMode, type SnapshotFreshnessWindow, @@ -96,11 +97,7 @@ export async function captureAndroidFreshnessRecoveredAttempt< { snapshot: attempt.snapshot, rawNodeCount: attempt.annotations.analysis?.rawNodeCount, - cacheInvalidated: - attempt.annotations.androidSnapshot?.cacheInvalidated === true && - attempt.annotations.androidSnapshot.rootPresent === true && - attempt.annotations.androidSnapshot.helperTruncated === false && - attempt.annotations.androidSnapshot.missingRootWindowTypes?.length === 0, + cacheInvalidated: provesFreshAndroidTraversal(attempt.annotations.androidSnapshot), }, freshness, { interactiveOnly: params.interactiveOnly, mode: params.androidFreshnessMode }, diff --git a/packages/daemon/src/core/snapshot-capture.ts b/packages/daemon/src/core/snapshot-capture.ts index cb1f412cca..d79c429fc7 100644 --- a/packages/daemon/src/core/snapshot-capture.ts +++ b/packages/daemon/src/core/snapshot-capture.ts @@ -26,7 +26,10 @@ import { INTERACTION_ERROR_REASONS } from '@agent-device/selectors/interaction-e import { captureSnapshotWithInteractor } from './snapshot-interactor-capture.ts'; import { buildSnapshotState } from '@agent-device/capture-kit/snapshot-state'; import { clearAndroidSnapshotFreshness } from './session-snapshot-freshness.ts'; -import type { SnapshotFreshnessMode } from '@agent-device/capture-kit/snapshot-freshness'; +import { + provesFreshAndroidTraversal, + type SnapshotFreshnessMode, +} from '@agent-device/capture-kit/snapshot-freshness'; import { contextFromFlags } from './context.ts'; import { resolveDeferredInteractionOutcome } from './deferred-interaction-outcome.ts'; import type { SessionState } from './session-state.ts'; @@ -149,6 +152,7 @@ async function captureSnapshotAttempt(params: CaptureSnapshotParams): Promise { + await capture(); + return { + settled: true, + waitedMs: 0, + captures: 2, + quietMs: settle.quietMs ?? 0, + timeoutMs: settle.timeoutMs ?? 0, + diff: { summary: { additions: 0, removals: 0, unchanged: 1 }, lines: [] }, + }; + }, +}); + +function continueCapture(missingRootWindowTypes: number[]) { + return { + nodes: [ + { + index: 0, + type: 'android.widget.Button', + label: 'Continue', + rect: { x: 100, y: 200, width: 80, height: 40 }, + enabled: true, + hittable: true, + }, + ], + backend: 'android' as const, + producer: 'android-uiautomator' as const, + androidSnapshot: { + backend: 'android-helper' as const, + cacheInvalidated: true, + rootPresent: true, + helperTruncated: false, + missingRootWindowTypes, + }, + }; +} + +// The resolution capture always carries the opposite proof, so only the settled capture decides. +test.each([ + { capture: 'proved fresh', outcome: 'retires', resolved: [2], settled: [], window: undefined }, + { + capture: 'missing a window root', + outcome: 'keeps', + resolved: [], + settled: [2], + window: expect.objectContaining({ action: 'press' }), + }, +])( + 'press --settle converged on a capture $capture $outcome the freshness window', + async ({ resolved, settled, window }) => { + const sessionStore = makeSessionStore(); + const sessionName = 'android-settle-freshness'; + sessionStore.publish(sessionName, makeAndroidSession(sessionName)); + mockInteractorSnapshot + .mockResolvedValueOnce(continueCapture(resolved)) + .mockResolvedValue(continueCapture(settled)); + + const response = await handleInteractionCommands({ + createCommandSurface: convergingSurface, + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=Continue'], + flags: { settle: true, settleQuietMs: 25, timeoutMs: 2_000 }, + }, + sessionName, + sessionStore, + contextFromFlags, + ...getRuntimeBindings(), + }); + + expect(response?.ok).toBe(true); + expect(mockInteractorSnapshot).toHaveBeenCalledTimes(2); + expect(sessionStore.get(sessionName)?.androidSnapshotFreshness).toEqual(window); + }, +); + test('ADR 0014: Android freshness cannot retarget an admitted ref by positional coincidence', async () => { const sessionStore = makeSessionStore(); const sessionName = 'android-fresh-ref-no-retarget';