diff --git a/docs/adr/0016-active-session-script-publication.md b/docs/adr/0016-active-session-script-publication.md index fef2b23920..9e448469ad 100644 --- a/docs/adr/0016-active-session-script-publication.md +++ b/docs/adr/0016-active-session-script-publication.md @@ -208,7 +208,8 @@ It adds no state name, command, or flag: - The daemon reports the `script-artifacts` capability in `/health`, through a proxy in its upstream health; the client refuses a remote `--save-script` before dispatch without it. The destination keeps local `--save-script` semantics: the client creates a missing parent directory and checks no ancestor ownership, so another local user who can write an ancestor directory can redirect the file, locally and remotely alike (#4565). -Known limitation (#4564): the RPC response is the contract for `scriptDeliveryFailure`; the SDK, CLI, and MCP results do not carry it yet, the SDK close drops its warnings, and a batch delivers no step script versions. +Every result that can carry a script version discloses a failed delivery: the SDK result, the CLI human output and `--json`, and the MCP tool result each carry `scriptDeliveryFailure` and the warning, so no caller reads a stale file as current. +Known limitation (#4595): a batch delivers no script versions from its steps, so closing an armed remote recording inside a batch stays refused. Rejected: a remote `session save-script` download, because it added a second publication path and a retry protocol. Rejected: delivery only at close, because a session that an external owner closes would lose its recording. Rejected: daemon-minted recording IDs bound through per-request claims, because the claim, binding, and tombstone protocol cost more than a client-minted ID that only its own client can match. diff --git a/packages/client/src/__tests__/index-script-delivery-disclosure.test.ts b/packages/client/src/__tests__/index-script-delivery-disclosure.test.ts new file mode 100644 index 0000000000..a5eee35285 --- /dev/null +++ b/packages/client/src/__tests__/index-script-delivery-disclosure.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../index.ts'; +import type { DaemonResponse, ScriptDeliveryFailure } from '@agent-device/kernel/contracts'; +import { createClientTransport } from '@agent-device/test-kit'; + +/** + * A script version that did not reach the caller file keeps its action successful, so the typed + * failure and its paired warning are the only disclosure that the file is stale (ADR 0016, #4564). + * Every SDK result that can carry a script version reads both fields through + * `readScriptDeliveryDisclosure`; this table pins each one, so a spread that stops carrying either + * field fails here rather than quietly on one surface. + */ + +const WARNING = 'The session recorded version 2, but the daemon could not save it.'; + +const FAILURE: ScriptDeliveryFailure = { + reason: 'script_delivery_failed', + cause: 'script_write_failed', + publicationId: '0f61b0c2-6d1a-4d1f-9b7f-2a4f3a1c7d55', + message: WARNING, + hint: 'The file keeps the version from the last action that arrived.', +}; + +/** The two fields one answer carries at the top level of its payload. */ +const DISCLOSURE = { warnings: [WARNING], scriptDeliveryFailure: FAILURE }; + +/** One SDK action whose answer can carry a script version, with the answer it reads. */ +type DisclosureSurface = Readonly<{ + label: string; + command: string; + data: Record; + read: (client: ReturnType) => Promise; +}>; + +const SURFACES: DisclosureSurface[] = [ + { + label: 'sessions.close', + command: 'close', + data: { ...DISCLOSURE, savedScript: '/tmp/journey.ad' }, + read: async (client) => await client.sessions.close(), + }, + { + label: 'apps.close', + command: 'close', + data: { ...DISCLOSURE, savedScript: '/tmp/journey.ad' }, + read: async (client) => await client.apps.close(), + }, + { + label: 'apps.open', + command: 'open', + data: { ...DISCLOSURE, appBundleId: 'com.example.demo' }, + read: async (client) => await client.apps.open({ app: 'Demo' }), + }, + { + label: 'capture.snapshot', + command: 'snapshot', + data: { ...DISCLOSURE, nodes: [] }, + read: async (client) => await client.capture.snapshot(), + }, + { + label: 'capture.screenshot', + command: 'screenshot', + data: { ...DISCLOSURE, path: '/tmp/shot.png', width: 40, height: 20 }, + read: async (client) => await client.capture.screenshot(), + }, +]; + +function clientAnswering(command: string, data: Record) { + const setup = createClientTransport(async (req): Promise => { + if (req.command !== command) throw new Error(`Unexpected command: ${req.command}`); + return { ok: true, data: { session: 'qa', ...data } }; + }); + return createAgentDeviceClient(setup.config, { transport: setup.transport }); +} + +for (const surface of SURFACES) { + test(`the ${surface.label} result carries a failed script delivery`, async () => { + const client = clientAnswering(surface.command, surface.data); + + const result = (await surface.read(client)) as Record; + + assert.deepEqual(result.scriptDeliveryFailure, FAILURE, surface.label); + assert.deepEqual(result.warnings, [WARNING], surface.label); + }); + + test(`the ${surface.label} result names no disclosure for a delivered script`, async () => { + const plain = Object.fromEntries( + Object.entries(surface.data).filter( + ([key]) => key !== 'warnings' && key !== 'scriptDeliveryFailure', + ), + ); + const client = clientAnswering(surface.command, plain); + + const result = (await surface.read(client)) as Record; + + assert.equal('scriptDeliveryFailure' in result, false, surface.label); + assert.equal('warnings' in result, false, surface.label); + }); + + test(`the ${surface.label} result drops a malformed failure object`, async () => { + const client = clientAnswering(surface.command, { + ...surface.data, + scriptDeliveryFailure: { reason: 'script_delivery_failed' }, + }); + + const result = (await surface.read(client)) as Record; + + assert.equal('scriptDeliveryFailure' in result, false, surface.label); + }); +} diff --git a/packages/client/src/client-normalizers.ts b/packages/client/src/client-normalizers.ts index 52b3439fda..65afa866e1 100644 --- a/packages/client/src/client-normalizers.ts +++ b/packages/client/src/client-normalizers.ts @@ -24,7 +24,13 @@ import { isSerialAddressablePlatform, type AppleOS, } from '@agent-device/kernel/device'; -import { isSessionRuntimePlatform, type SessionRuntimeHints } from '@agent-device/kernel/contracts'; +import { + isSessionRuntimePlatform, + readScriptDeliveryFailure, + type ScriptDeliveryFailure, + type SessionRuntimeHints, +} from '@agent-device/kernel/contracts'; +import { readResponseWarnings } from '@agent-device/kernel/success-text'; import { AppError, type DaemonError } from '@agent-device/kernel/errors'; import { sanitizeErrorCause } from '@agent-device/kernel/redaction'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; @@ -486,3 +492,24 @@ export function buildMeta(options: InternalRequestOptions): DaemonWireRequest['m export function resolveSessionName(session: string | undefined): string { return session ?? DEFAULT_SESSION_NAME; } + +/** + * The disclosure fields a successful action carries when a remote recording's script version did not + * reach its caller file (ADR 0016, #4564). + * + * A script delivery failure never changes the outcome of the action that recorded the version, so + * the response stays successful and these two fields are the only channel that says the file is + * stale. Every SDK result that can carry a script version reads them through this one owner, so no + * surface can drop the disclosure while another keeps it. + */ +export function readScriptDeliveryDisclosure(data: Record): { + warnings?: string[]; + scriptDeliveryFailure?: ScriptDeliveryFailure; +} { + const warnings = readResponseWarnings(data); + const scriptDeliveryFailure = readScriptDeliveryFailure(data); + return { + ...(warnings.length > 0 ? { warnings } : {}), + ...(scriptDeliveryFailure === undefined ? {} : { scriptDeliveryFailure }), + }; +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index f9be775e68..b00e671665 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -49,6 +49,7 @@ import { normalizeTargetShutdownResult, readOptionalString, readRequiredString, + readScriptDeliveryDisclosure, readSnapshotNodes, resolveSessionName, } from './client-normalizers.ts'; @@ -70,7 +71,6 @@ import { readSnapshotKeyboardBandFact, } from '@agent-device/kernel/record'; import { readSnapshotViewportSize } from '@agent-device/kernel/rect'; -import { readResponseWarnings } from '@agent-device/kernel/success-text'; import { captureHostEnvironment } from '@agent-device/host-kit/process'; import type { HostEnvironment } from '@agent-device/kernel/source-value'; import { createLeaseClient } from './lease-client.ts'; @@ -273,6 +273,7 @@ export function createAgentDeviceClient( shutdown: normalizeTargetShutdownResult(data.shutdown), provider: asOptionalRecord(data.provider), savedScript: readOptionalString(data, 'savedScript'), + ...readScriptDeliveryDisclosure(data), identifiers: { session }, }; } finally { @@ -345,10 +346,9 @@ export function createAgentDeviceClient( const device = normalizeOpenDevice(data); const appBundleId = readOptionalString(data, 'appBundleId'); const appId = appBundleId; - const warnings = readResponseWarnings(data); return { session, - ...(warnings.length > 0 ? { warnings } : {}), + ...readScriptDeliveryDisclosure(data), sessionStateDir: readOptionalString(data, 'sessionStateDir'), runnerLogPath: readOptionalString(data, 'runnerLogPath'), requestLogPath: readOptionalString(data, 'requestLogPath'), @@ -381,6 +381,7 @@ export function createAgentDeviceClient( closedApp: options.app, shutdown: normalizeTargetShutdownResult(data.shutdown), savedScript: readOptionalString(data, 'savedScript'), + ...readScriptDeliveryDisclosure(data), identifiers: { session }, }; }, @@ -605,6 +606,7 @@ function optionalSnapshotResponseFields( const snapshotDiagnostics = readSnapshotDiagnosticsSummary(data.snapshotDiagnostics); const presentedSurface = readPresentedSurfaceFact(data); const postGestureOutcome = readPostGestureOutcome(data.postGestureOutcome); + const { scriptDeliveryFailure } = readScriptDeliveryDisclosure(data); return { ...(keyboard ? { keyboard } : {}), ...(viewport ? { viewport } : {}), @@ -614,6 +616,9 @@ function optionalSnapshotResponseFields( ...readSerializedSnapshotCaptureAnnotations(data), ...(unchanged ? { unchanged: unchanged as CaptureSnapshotResult['unchanged'] } : {}), ...(snapshotDiagnostics ? { snapshotDiagnostics } : {}), + // `readSerializedSnapshotCaptureAnnotations` above already reads this response's `warnings`, so + // only the typed field needs adding here. + ...(scriptDeliveryFailure === undefined ? {} : { scriptDeliveryFailure }), ...(typeof data.fallbackScreenshotPath === 'string' ? { fallbackScreenshotPath: data.fallbackScreenshotPath } : {}), diff --git a/packages/client/src/screenshot-result.ts b/packages/client/src/screenshot-result.ts index 210c685066..d49a9fa86c 100644 --- a/packages/client/src/screenshot-result.ts +++ b/packages/client/src/screenshot-result.ts @@ -1,6 +1,7 @@ import type { ScreenshotResultData } from '@agent-device/contracts/capture'; import type { CaptureScreenshotResult } from '@agent-device/contracts/client'; import { isDeviceRotation } from '@agent-device/contracts/device'; +import { readScriptDeliveryDisclosure } from './client-normalizers.ts'; import { isRecord, parsePoint, parseRect, readRequiredString } from '@agent-device/kernel/record'; import { readResponseWarnings } from '@agent-device/kernel/success-text'; import type { ScreenshotOverlayRef } from '@agent-device/kernel/snapshot'; @@ -25,6 +26,7 @@ export function normalizeScreenshotCaptureResult( session: string, ): CaptureScreenshotResult { const screenshot = readScreenshotResultData(data); + const { scriptDeliveryFailure } = readScriptDeliveryDisclosure(data); return { path: readRequiredString(data, 'path'), width: screenshot?.width, @@ -34,7 +36,10 @@ export function normalizeScreenshotCaptureResult( pixelDensity: screenshot?.pixelDensity, ...(screenshot?.displayRotation ? { displayRotation: screenshot.displayRotation } : {}), overlayRefs: screenshot?.overlayRefs, + // `screenshot.warnings` is read from this same response, so it already carries the warning a + // failed script delivery adds. Only the typed field needs adding here. ...(screenshot?.warnings ? { warnings: screenshot.warnings } : {}), + ...(scriptDeliveryFailure ? { scriptDeliveryFailure } : {}), identifiers: { session }, }; } diff --git a/packages/command-output/src/output/result-serialization-script-delivery.test.ts b/packages/command-output/src/output/result-serialization-script-delivery.test.ts new file mode 100644 index 0000000000..47ebd67580 --- /dev/null +++ b/packages/command-output/src/output/result-serialization-script-delivery.test.ts @@ -0,0 +1,72 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { ScriptDeliveryFailure } from '@agent-device/kernel/contracts'; +import { + serializeCloseResult, + serializeOpenResult, + serializeSnapshotResult, +} from './result-serialization.ts'; + +/** + * A remote recording's script version that did not reach the caller file keeps its action + * successful, so machine output is where a caller learns the file is stale (ADR 0016, #4564). Every + * family that can report a script version emits the two fields through one helper; this table pins + * each of them, so a serializer that stops carrying either field fails here. + */ + +const WARNING = 'The session recorded version 2, but the daemon could not save it.'; + +const FAILURE: ScriptDeliveryFailure = { + reason: 'script_delivery_failed', + cause: 'script_write_failed', + publicationId: '0f61b0c2-6d1a-4d1f-9b7f-2a4f3a1c7d55', + message: WARNING, + hint: 'The file keeps the version from the last action that arrived.', +}; + +const DISCLOSURE = { warnings: [WARNING], scriptDeliveryFailure: FAILURE }; + +/** The machine output of each family whose result can carry a script version. */ +const outputs: Array<{ + label: string; + /** Called with the disclosure fields, and with PLAIN input that names none of them. */ + serialize: (input?: Record) => Record; +}> = [ + { + label: 'open', + serialize: (input) => serializeOpenResult({ session: 'qa', ...input } as never), + }, + { + label: 'session close', + serialize: (input) => + serializeCloseResult({ session: 'qa', savedScript: '/tmp/journey.ad', ...input } as never), + }, + { + label: 'snapshot', + serialize: (input) => serializeSnapshotResult({ nodes: [], ...input } as never), + }, +]; + +for (const output of outputs) { + test(`the ${output.label} machine output carries a failed script delivery`, () => { + const data = output.serialize(DISCLOSURE); + + assert.deepEqual(data.scriptDeliveryFailure, FAILURE); + assert.deepEqual(data.warnings, [WARNING]); + }); + + test(`the ${output.label} machine output omits a disclosure it was not given`, () => { + // PLAIN input: nothing here names a script, so a serializer that invents either field fails. + const data = output.serialize(); + + assert.equal('scriptDeliveryFailure' in data, false); + assert.equal('warnings' in data, false); + }); + + test(`the ${output.label} machine output keeps the failure beside an empty warning list`, () => { + const empty = output.serialize({ ...DISCLOSURE, warnings: [] }); + + assert.deepEqual(empty.scriptDeliveryFailure, FAILURE); + assert.equal('warnings' in empty, false); + }); +} diff --git a/packages/command-output/src/output/result-serialization.ts b/packages/command-output/src/output/result-serialization.ts index cc7d5e1fba..f02655b7ac 100644 --- a/packages/command-output/src/output/result-serialization.ts +++ b/packages/command-output/src/output/result-serialization.ts @@ -10,6 +10,7 @@ import type { SessionCloseResult, } from '@agent-device/contracts/client'; import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { ScriptDeliveryFailure } from '@agent-device/kernel/contracts'; import { publicSnapshotCaptureAnnotations } from '@agent-device/contracts/capture'; import { isSerialAddressablePlatform } from '@agent-device/kernel/device'; import { successText, withSuccessText } from '@agent-device/kernel/success-text'; @@ -92,6 +93,9 @@ export function serializeSnapshotResult(result: CaptureSnapshotResult): Record { + return { + ...(result.warnings?.length ? { warnings: result.warnings } : {}), + ...(result.scriptDeliveryFailure + ? { scriptDeliveryFailure: result.scriptDeliveryFailure } + : {}), + }; +} + export function serializeCloseResult( result: SessionCloseResult | AppCloseResult, ): Record { @@ -179,6 +200,7 @@ export function serializeCloseResult( session: result.session, ...(result.shutdown ? { shutdown: result.shutdown } : {}), ...('provider' in result && result.provider ? { provider: result.provider } : {}), + ...serializeScriptDeliveryDisclosure(result), ...successText( [ result.session ? `Closed: ${result.session}` : 'Closed', diff --git a/packages/contracts/src/client-app.ts b/packages/contracts/src/client-app.ts index a462387ed7..9a2ca879d2 100644 --- a/packages/contracts/src/client-app.ts +++ b/packages/contracts/src/client-app.ts @@ -8,7 +8,11 @@ import type { import type { JsonObject } from './json.ts'; import type { SessionSurface } from './session-surface.ts'; import type { TargetShutdownResult } from './target-shutdown-contract.ts'; -import type { DaemonInstallSource, SessionRuntimeHints } from '@agent-device/kernel/contracts'; +import type { + DaemonInstallSource, + ScriptDeliveryFailure, + SessionRuntimeHints, +} from '@agent-device/kernel/contracts'; import type { DaemonError } from '@agent-device/kernel/errors'; import type { PublicPlatform } from '@agent-device/kernel/device'; import type { @@ -113,6 +117,11 @@ export type AppOpenResult = AppOpenResultBase & type AppOpenResultBase = { session: string; warnings?: string[]; + /** + * ADR 0016 (#4564): a remote recording's first script version was not delivered. The open kept + * its success, so the caller file may hold no script at all, or an older one. + */ + scriptDeliveryFailure?: ScriptDeliveryFailure; sessionStateDir?: string; runnerLogPath?: string; requestLogPath?: string; @@ -161,6 +170,13 @@ export type AppCloseResult = { * — so a client that requested publication learns where the file landed. */ savedScript?: string; + /** ADR 0016 (#4564): the close's response warnings, kept so a lost final script is read, not skipped. */ + warnings?: string[]; + /** + * ADR 0016 (#4564): the final script of a remote recording was not delivered. The close kept its + * success, so this is the only channel that says the caller file still holds an older version. + */ + scriptDeliveryFailure?: ScriptDeliveryFailure; identifiers: AgentDeviceIdentifiers; }; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 1afdf01cf2..db9f48ad04 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -13,6 +13,7 @@ import type { SnapshotViewportSize, SnapshotVisibility, } from '@agent-device/kernel/snapshot'; +import type { ScriptDeliveryFailure } from '@agent-device/kernel/contracts'; import type { ScreenshotResultData } from './snapshot-types.ts'; import type { PostGestureOutcome } from './snapshot-model.ts'; import type { @@ -108,6 +109,11 @@ export type CaptureSnapshotResult = { * default-level `nodes` payload is intentionally omitted. */ refs?: Array<{ ref: string; label?: string }>; + /** + * ADR 0016 (#4564): a remote recording's script version this capture recorded was not delivered. + * The capture kept its success, so the caller file may hold an older version. + */ + scriptDeliveryFailure?: ScriptDeliveryFailure; } & PublicSnapshotCaptureAnnotations; export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { @@ -128,6 +134,11 @@ export type CaptureScreenshotOptions = AgentDeviceRequestOverrides & { export type CaptureScreenshotResult = ScreenshotResultData & { path: string; + /** + * ADR 0016 (#4564): a remote recording's script version this capture recorded was not delivered. + * The capture kept its success, so the caller file may hold an older version. + */ + scriptDeliveryFailure?: ScriptDeliveryFailure; identifiers: AgentDeviceIdentifiers; }; diff --git a/packages/contracts/src/client-session.ts b/packages/contracts/src/client-session.ts index 22e2e91be2..c44d143c56 100644 --- a/packages/contracts/src/client-session.ts +++ b/packages/contracts/src/client-session.ts @@ -3,6 +3,7 @@ import type { CloudProviderSessionResult } from './cloud-artifacts.ts'; import type { TargetShutdownResult } from './target-shutdown-contract.ts'; import type { AgentDeviceIdentifiers, AgentDeviceRequestOverrides } from './client-connection.ts'; +import type { ScriptDeliveryFailure } from '@agent-device/kernel/contracts'; export type SessionCloseResult = { session: string; @@ -14,6 +15,13 @@ export type SessionCloseResult = { * — so a client that requested publication learns where the file landed. */ savedScript?: string; + /** ADR 0016 (#4564): the close's response warnings, kept so a lost final script is read, not skipped. */ + warnings?: string[]; + /** + * ADR 0016 (#4564): the final script of a remote recording was not delivered. The close kept its + * success, so this is the only channel that says the caller file still holds an older version. + */ + scriptDeliveryFailure?: ScriptDeliveryFailure; identifiers: AgentDeviceIdentifiers; }; diff --git a/packages/kernel/src/contracts.ts b/packages/kernel/src/contracts.ts index 9aa591d1dc..c352cdeaa4 100644 --- a/packages/kernel/src/contracts.ts +++ b/packages/kernel/src/contracts.ts @@ -222,6 +222,29 @@ export type ScriptDeliveryFailure = { cause?: string; }; +/** + * Reads a response's `scriptDeliveryFailure` (ADR 0016). A response that carries no such record, or + * one the field contract does not describe, answers `undefined`: no surface acts on a half-shaped + * failure, and every surface that shows one shows it through this reader. + */ +export function readScriptDeliveryFailure( + data: Record | undefined, +): ScriptDeliveryFailure | undefined { + const failure = data?.scriptDeliveryFailure; + if (!failure || typeof failure !== 'object' || Array.isArray(failure)) return undefined; + const { reason, publicationId, message, hint, cause } = failure as Partial; + if ( + reason !== 'script_delivery_failed' || + typeof publicationId !== 'string' || + typeof message !== 'string' || + typeof hint !== 'string' || + (cause !== undefined && typeof cause !== 'string') + ) { + return undefined; + } + return { reason, publicationId, message, hint, ...(cause === undefined ? {} : { cause }) }; +} + export type ResponseCost = { wallClockMs: number; // Number of real iOS-runner round-trips made while serving the request (the diff --git a/src/cli/commands/__tests__/screenshot.test.ts b/src/cli/commands/__tests__/screenshot.test.ts index 626b916b45..dece774ae0 100644 --- a/src/cli/commands/__tests__/screenshot.test.ts +++ b/src/cli/commands/__tests__/screenshot.test.ts @@ -77,6 +77,37 @@ test('screenshot --json carries response-level warnings in the data', async () = assert.deepEqual(parsed.data.warnings, [CROP_WARNING]); }); +// ADR 0016 (#4564): a capture whose remote script version did not arrive keeps its success, so the +// typed failure has to ride the machine view this command prints. `pickScreenshotResultData` narrows +// the capture payload for its own callers, so this command adds the disclosure itself. +test('screenshot --json carries a failed script delivery of the capture', async () => { + const failure = { + reason: 'script_delivery_failed', + cause: 'script_write_failed', + publicationId: '0f61b0c2-6d1a-4d1f-9b7f-2a4f3a1c7d55', + message: 'The session recorded version 2, but the daemon could not save it.', + hint: 'The file keeps the version from the last action that arrived.', + }; + const client = clientReturning({ + path: '/tmp/shot.png', + width: 40, + height: 20, + warnings: [failure.message], + scriptDeliveryFailure: failure, + }); + const flags = { json: true } as CliFlags; + + const out = await withCapturedStdoutAsync(() => + screenshotCommand({ host: createTestHostEnvironment(), positionals: [], flags, client }), + ); + const parsed = JSON.parse(out) as { + data: { path: string; warnings?: string[]; scriptDeliveryFailure?: unknown }; + }; + + assert.deepEqual(parsed.data.scriptDeliveryFailure, failure); + assert.deepEqual(parsed.data.warnings, [failure.message]); +}); + test('screenshot --json at the default level still emits normalized screenshot metadata', async () => { const full = { path: '/tmp/shot.png', diff --git a/src/cli/commands/screenshot.ts b/src/cli/commands/screenshot.ts index ec43610e0e..c1f7b32e85 100644 --- a/src/cli/commands/screenshot.ts +++ b/src/cli/commands/screenshot.ts @@ -33,7 +33,14 @@ export const screenshotCommand: ClientCommandHandler = async ({ await writeCommandOutput(flags, result, () => JSON.stringify(result, null, 2)); return true; } - const data = pickScreenshotResultData(result); + // The disclosure fields (ADR 0016, #4564) sit beside the capture fields on the result, and + // `pickScreenshotResultData` narrows the capture payload for its own callers, so `--json` adds them. + const data = { + ...pickScreenshotResultData(result), + ...(result.scriptDeliveryFailure + ? { scriptDeliveryFailure: result.scriptDeliveryFailure } + : {}), + }; await writeCommandOutput(flags, data, () => { const summary = result.overlayRefs ? `Annotated ${result.overlayRefs.length} refs onto ${result.path}` diff --git a/test/integration/provider-scenarios/remote-script-publication.test.ts b/test/integration/provider-scenarios/remote-script-publication.test.ts index 6172e98640..22249484d7 100644 --- a/test/integration/provider-scenarios/remote-script-publication.test.ts +++ b/test/integration/provider-scenarios/remote-script-publication.test.ts @@ -12,6 +12,9 @@ import { assertRpcOk } from './assertions.ts'; import { createAndroidSettingsWorld } from './android-world.ts'; import { withProviderScenarioResource } from './harness.ts'; import { createProviderScenarioProxy } from './harness-proxy.ts'; +import { runCliCapture } from '../../../src/__tests__/cli-capture.ts'; +import { installIsolatedCliTestEnv } from '../../../src/__tests__/cli-test-env.ts'; +import { createCommandToolExecutor } from '../../../src/mcp/command-tools.ts'; /** A script version the daemon writes in its private directory. */ function isDaemonScriptPath(file: fs.PathLike): boolean { @@ -255,7 +258,7 @@ test('a failed final write keeps the close and the client forgets the recording' return realLinkSync(existing, target); }); - await client.sessions.close(); + const closeResult = await client.sessions.close(); const reply = proxy.forwarded.filter((r) => r.route === '/rpc').at(-1); assert.ok(reply); @@ -268,6 +271,15 @@ test('a failed final write keeps the close and the client forgets the recording' warning.includes(closed.data.scriptDeliveryFailure.hint), ), ); + // The SDK result carries the same typed failure and warning the wire answers, so a caller that + // never reads stdout sees the stale file. + assert.deepEqual(closeResult.scriptDeliveryFailure, closed.data.scriptDeliveryFailure); + assert.ok( + closeResult.warnings?.some((warning) => + warning.includes(closed.data.scriptDeliveryFailure.hint), + ), + JSON.stringify(closeResult.warnings), + ); assert.equal(fs.readFileSync(destination, 'utf8'), opened); const records = path.join(stateDir, 'remote-connections', 'session-scripts'); assert.deepEqual( @@ -438,3 +450,197 @@ test('close delivers the final script after it stops an app log', async () => { } }); }, 20_000); + +/** + * The caller surfaces a remote recording reads (#4564): an action whose script version did not + * arrive stays successful, so the SDK result, the CLI output and the MCP tool result are the only + * channels that say the caller file is stale. One gate refuses the artifact downloads of one + * surface's close at a time, so each surface answers its own failing delivery. + */ +test('the SDK, the CLI and MCP each disclose a failed final script delivery', async () => { + await withProviderScenarioResource(createAndroidSettingsWorld, async (world) => { + const root = mkdtempForTestSync('remote-script-surfaces-'); + const proxy = await createProviderScenarioProxy(world.daemon, { clientToken: 'script-client' }); + let refuseArtifacts = false; + const gate = await createArtifactGate(proxy.baseUrl, () => refuseArtifacts); + const remoteEnv = { + AGENT_DEVICE_DAEMON_BASE_URL: gate.baseUrl, + AGENT_DEVICE_DAEMON_AUTH_TOKEN: 'script-client', + AGENT_DEVICE_DAEMON_TRANSPORT: 'http', + }; + const recordFiles = (stateDir: string) => + fs + .readdirSync(path.join(stateDir, 'remote-connections', 'session-scripts'), { + withFileTypes: true, + }) + .filter((entry) => entry.isFile() && entry.name.endsWith('.json')) + .map((entry) => entry.name); + try { + // The SDK: the typed failure and the warning ride the close result itself. + const sdkStateDir = path.join(root, 'sdk-state'); + const sdkFile = path.join(root, 'sdk.ad'); + const sdk = createAgentDeviceClient({ + daemonBaseUrl: gate.baseUrl, + daemonAuthToken: 'script-client', + session: 'sdk-surfaces', + stateDir: sdkStateDir, + }); + await sdk.apps.open({ app: 'settings', saveScript: sdkFile, ...world.selection }); + refuseArtifacts = true; + const sdkClose = await sdk.sessions.close(); + refuseArtifacts = false; + assert.equal(sdkClose.scriptDeliveryFailure?.reason, 'script_delivery_failed'); + assert.ok( + sdkClose.warnings?.some((warning) => + warning.includes(sdkClose.scriptDeliveryFailure!.hint), + ), + JSON.stringify(sdkClose.warnings), + ); + assert.doesNotMatch(fs.readFileSync(sdkFile, 'utf8'), /^close/m); + assert.deepEqual(recordFiles(sdkStateDir), []); + + // The CLI with `--json`: an agent reading machine output gets the typed field. + const jsonStateDir = path.join(root, 'cli-json-state'); + const jsonFile = path.join(root, 'cli-json.ad'); + const cli = (argv: string[], stateDir: string) => + runCliCapture(argv, { + cwd: root, + env: { ...remoteEnv, AGENT_DEVICE_STATE_DIR: stateDir }, + useRealDaemonClient: true, + }); + const jsonOpen = await cli( + [ + 'open', + 'settings', + '--json', + '--session', + 'cli-json', + '--save-script', + jsonFile, + '--platform', + world.selection.platform, + '--serial', + world.selection.serial, + ], + jsonStateDir, + ); + assert.equal(jsonOpen.code, null, jsonOpen.stderr); + refuseArtifacts = true; + const jsonClose = await cli(['close', '--json', '--session', 'cli-json'], jsonStateDir); + refuseArtifacts = false; + assert.equal(jsonClose.code, null, jsonClose.stderr); + const jsonPayload = JSON.parse(jsonClose.stdout); + assert.equal(jsonPayload.success, true, jsonClose.stdout); + assert.equal(jsonPayload.data.scriptDeliveryFailure.reason, 'script_delivery_failed'); + assert.ok( + Array.isArray(jsonPayload.data.warnings) && + jsonPayload.data.warnings.some((warning: string) => + warning.includes(jsonPayload.data.scriptDeliveryFailure.hint), + ), + jsonClose.stdout, + ); + assert.doesNotMatch(fs.readFileSync(jsonFile, 'utf8'), /^close/m); + + // The CLI for a human: the warning is a line the terminal prints. + const humanStateDir = path.join(root, 'cli-human-state'); + const humanFile = path.join(root, 'cli-human.ad'); + const humanOpen = await cli( + [ + 'open', + 'settings', + '--session', + 'cli-human', + '--save-script', + humanFile, + '--platform', + world.selection.platform, + '--serial', + world.selection.serial, + ], + humanStateDir, + ); + assert.equal(humanOpen.code, null, humanOpen.stderr); + refuseArtifacts = true; + const humanClose = await cli(['close', '--session', 'cli-human'], humanStateDir); + refuseArtifacts = false; + assert.equal(humanClose.code, null, humanClose.stderr); + assert.match( + `${humanClose.stdout}${humanClose.stderr}`, + /Warning: .*remote script could not be saved/i, + ); + + // MCP: the tool text the model reads carries the warning, and machine output carries the field. + const mcpStateDir = path.join(root, 'mcp-state'); + const mcpFile = path.join(root, 'mcp.ad'); + const restoreEnv = installIsolatedCliTestEnv({ + ...remoteEnv, + AGENT_DEVICE_STATE_DIR: mcpStateDir, + }); + try { + const executor = createCommandToolExecutor({ structuredContent: 'always' }); + const mcpOpen = await executor.execute('open', { + app: 'settings', + session: 'mcp-surfaces', + saveScript: mcpFile, + ...world.selection, + }); + assert.equal(mcpOpen.isError, false, JSON.stringify(mcpOpen)); + refuseArtifacts = true; + const mcpClose = await executor.execute('close', { session: 'mcp-surfaces' }); + refuseArtifacts = false; + assert.equal(mcpClose.isError, false, JSON.stringify(mcpClose)); + assert.match( + mcpClose.content.map((part) => part.text ?? '').join('\n'), + /Warning: .*remote script could not be saved/i, + ); + assert.equal( + (mcpClose.structuredContent as { scriptDeliveryFailure?: { reason?: string } }) + ?.scriptDeliveryFailure?.reason, + 'script_delivery_failed', + ); + } finally { + restoreEnv(); + } + assert.doesNotMatch(fs.readFileSync(mcpFile, 'utf8'), /^close/m); + world.assertNoHostAdbCalls(); + } finally { + await gate.close(); + await proxy.close(); + } + }); +}, 30_000); +/** + * A gate in front of the daemon proxy that answers the artifact downloads of a refused moment with + * 503, which is how these scenarios make one delivery fail. + */ +async function createArtifactGate( + upstreamUrl: string, + refused: () => boolean, +): Promise<{ baseUrl: string; close(): Promise }> { + const upstream = new URL(upstreamUrl); + const gate = http.createServer((req, res) => { + if (refused() && req.url?.includes('/artifacts/')) { + res.writeHead(503).end('unavailable'); + return; + } + const forwarded = http.request( + { + host: upstream.hostname, + port: upstream.port, + method: req.method, + path: req.url, + headers: req.headers, + }, + (reply) => { + res.writeHead(reply.statusCode ?? 502, reply.headers); + reply.pipe(res); + }, + ); + forwarded.on('error', () => res.destroy()); + req.pipe(forwarded); + }); + return { + baseUrl: `http://127.0.0.1:${await listenOnLoopback(gate)}${upstream.pathname}`, + close: () => closeLoopbackServer(gate), + }; +} diff --git a/test/wire-compat/ledger.json b/test/wire-compat/ledger.json index 0eba21ea2f..bdabba0ca5 100644 --- a/test/wire-compat/ledger.json +++ b/test/wire-compat/ledger.json @@ -41,6 +41,7 @@ "packages/kernel/src/contracts.ts#commandRpcParamsSchema": "sha256:d0ce05c236ce5b0b52631ab7b8a75d38859fbfba58cfb28dd3f9d84f2dba2c1f", "packages/kernel/src/contracts.ts#daemonRuntimeSchema": "sha256:3b99926ce9deb5d66186a55b1168011dc1ae4d010ada5ac5d79c4c7225a1f80f", "packages/kernel/src/contracts.ts#jsonRpcRequestSchema": "sha256:67e6b8a28b39a3883424a565ae7033c336dc2ea6b193e9b32bb98eb3e18dca7c", + "packages/kernel/src/contracts.ts#readScriptDeliveryFailure": "sha256:58e1f8cbe074e05fe8a273ed05fb4bd9e1fa03a227939d233301fbd9e8a8ac77", "packages/kernel/src/device.ts#PLATFORM_SELECTORS": "sha256:36e9da1cc660c0ddfb4cf52521f7f230341ff20e3ebda3405ba7c1799476c42d", "packages/kernel/src/device.ts#PlatformSelector": "sha256:61de3f003507ea2f53b396b0146c17670bf674a2db2132e19c462ca6c10cc8fd", "packages/kernel/src/errors.ts#DaemonError": "sha256:b392b10f461b774f82252f8a5958781450b101329d43827abca10b6511b6d588", diff --git a/test/wire-compat/surface.ts b/test/wire-compat/surface.ts index 4d39df06b4..323fd76881 100644 --- a/test/wire-compat/surface.ts +++ b/test/wire-compat/surface.ts @@ -294,6 +294,7 @@ export const WIRE_SURFACE: readonly WireSurfaceGroup[] = [ 'DaemonArtifactType', 'DaemonArtifactKnownType', 'ScriptDeliveryFailure', + 'readScriptDeliveryFailure', ), ...from( KERNEL_ERRORS, diff --git a/website/docs/docs/remote-proxy.md b/website/docs/docs/remote-proxy.md index 2cfffa7015..44cd873478 100644 --- a/website/docs/docs/remote-proxy.md +++ b/website/docs/docs/remote-proxy.md @@ -117,14 +117,14 @@ A script version that is not delivered never fails the command that recorded it. fail to write it, or this machine may fail to download or save it, for example when it is larger than 8 MiB. The command still succeeds, and the file may keep the previous version; the next recorded action or `close` delivers the full recording again. If `close` cannot deliver the final script, -the session still closes and the file keeps the version from the last action. The daemon RPC -response carries a typed `scriptDeliveryFailure` and a warning. The CLI, SDK, and MCP results do -not carry `scriptDeliveryFailure` yet, and `close` shows no warning; see -[#4564](https://github.com/callstack/agent-device/issues/4564). +the session still closes and the file keeps the version from the last action. Every result that +recorded a version discloses it: the CLI prints a warning and carries a typed +`scriptDeliveryFailure` in `--json`, and the SDK and MCP results carry the same field. Remote `session save-script`, `close --save-script`, `replay --save-script`, and `--save-script` in batch steps are refused. A `batch` whose steps would close the recording is refused before any step -runs; run `close` on its own after the batch. +runs; run `close` on its own after the batch. A batch delivers no script versions from its steps +([#4595](https://github.com/callstack/agent-device/issues/4595)). Direct RPC callers send `saveScript: true` with a random `meta.scriptPublicationId` of 16 to 64 letters, digits, or hyphens. Each script artifact carries that `scriptPublicationId` and a diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 5a96d58ea4..d34e6eb31c 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -74,8 +74,9 @@ agent-device open Settings --platform ios --session e2e --save-script ./workflow - With a local daemon, the script is written on the daemon host when the session closes. - With a remote daemon, outside a macOS app lease, the path is on your machine. The file appears when `open` returns, and each later recorded action in the session replaces it with the recording so far. `close` writes the final script there. A session that another process closes keeps the version from the last action. - A bare remote `open --save-script` writes `recording-.ad` in your working directory. -- If the file already exists, `open` fails unless you pass `--force`. Without `--force`, a file that appears before the first version arrives is kept too: `open` still succeeds, but the script is not saved there. The daemon RPC response carries a `scriptDeliveryFailure` and a warning; the CLI, SDK, and MCP results do not carry the failure yet ([#4564](https://github.com/callstack/agent-device/issues/4564)). With `--force`, the first version replaces it. After that, the session owns the file and replaces it on each update. +- If the file already exists, `open` fails unless you pass `--force`. Without `--force`, a file that appears before the first version arrives is kept too: `open` still succeeds, but the script is not saved there, and the result carries a `scriptDeliveryFailure` and a warning. With `--force`, the first version replaces it. After that, the session owns the file and replaces it on each update. - Remote `session save-script`, `close --save-script`, `replay --save-script`, and `--save-script` in batch steps are refused. +- Steps of a remote `batch` do not update the file. A batch that would close a remote recording is refused; run `close` on its own after the batch. Batch delivery is tracked in [#4595](https://github.com/callstack/agent-device/issues/4595). - The remote daemon must report the `script-artifacts` capability in its `/health` response. Through a proxy, the upstream daemon's health response must report it. Otherwise `open --save-script` fails before the app opens. - For a bare file name that could be read as another argument, use `--save-script=workflow.ad` or a path-like value such as `./workflow.ad`.