From e32b4be686b248fdfeb103b5726a5cc980612a60 Mon Sep 17 00:00:00 2001 From: Tom Riglar Date: Mon, 26 Oct 2026 13:09:51 +0000 Subject: [PATCH] feat(device): inspect --all, element state, and screen text on a failed run `dcd device inspect` now leaves out the on-screen keyboard and System UI by default (the API does this), says when a keyboard is open and how many elements it left out, and `--all` shows them. Elements show checked, selected, focused and disabled. A failed `dcd device run` prints the app's visible text after the failure. Co-Authored-By: Claude Opus 5.5 --- src/commands/device.ts | 26 ++++++++++++++++++++++-- src/gateways/device-gateway.ts | 3 ++- src/types/domain/device-session.types.ts | 11 ++++++++++ src/utils/device-run.ts | 14 +++++++++++++ test/unit/device-command.test.ts | 20 ++++++++++++++++++ test/unit/device-run.test.ts | 11 ++++++++++ 6 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/commands/device.ts b/src/commands/device.ts index 43a371a..c30fc5d 100644 --- a/src/commands/device.ts +++ b/src/commands/device.ts @@ -710,13 +710,21 @@ const modelsSub = defineCommand({ const inspectSub = defineCommand({ meta: { name: 'inspect', description: 'The screen as compact JSON: each element with text, an id or a label' }, - args: { ...commonArgs, id: idArg }, + args: { + ...commonArgs, + id: idArg, + all: { + type: 'boolean', + default: false, + description: 'Also list the keyboard keys and the System UI (status and navigation bars)', + }, + }, async run({ args }) { const json = Boolean(args.json); await runDeviceCommand(json, async () => { const ctx = await context(args); const id = await resolveSessionId(ctx.apiUrl, ctx.auth, args.id as string | undefined); - const { screen, session } = await DeviceGateway.inspect(ctx.apiUrl, ctx.auth, id); + const { screen, session } = await DeviceGateway.inspect(ctx.apiUrl, ctx.auth, id, Boolean(args.all)); return { session, data: { screen }, @@ -728,6 +736,15 @@ const inspectSub = defineCommand({ ), ); logger.log(ui.branch(screen.elements.map((el) => formatElement(el)))); + const notes: string[] = []; + if (screen.keyboard_open) notes.push('A keyboard is open.'); + if (screen.hidden && (screen.hidden.keyboard > 0 || screen.hidden.system > 0)) { + notes.push( + `Left out ${screen.hidden.keyboard} keyboard and ${screen.hidden.system} System UI elements (--all shows them).`, + ); + } + + if (notes.length > 0) logger.log(ui.note(notes.join(' '))); }, }; }); @@ -741,6 +758,11 @@ export function formatElement(el: DeviceScreenElement): string { if (el.a11y && el.a11y !== el.txt) parts.push(`a11y:"${el.a11y}"`); if (el.rid) parts.push(`id:${el.rid}`); if (el.hint) parts.push(`hint:"${el.hint}"`); + for (const state of ['checked', 'selected', 'focused'] as const) { + if (el[state]) parts.push(state); + } + + if (el.enabled === false) parts.push('disabled'); return `${parts.join(' ')} ${colors.dim(el.b)}`; } diff --git a/src/gateways/device-gateway.ts b/src/gateways/device-gateway.ts index 25ce7fc..74b7277 100644 --- a/src/gateways/device-gateway.ts +++ b/src/gateways/device-gateway.ts @@ -167,8 +167,9 @@ export const DeviceGateway = { baseUrl: string, auth: AuthContext, id: string, + all = false, ): Promise<{ screen: DeviceScreen; session: DeviceSession }> { - return request(baseUrl, auth, 'GET', `${sessionPath(id)}/inspect`); + return request(baseUrl, auth, 'GET', `${sessionPath(id)}/inspect${all ? '?all=true' : ''}`); }, install( diff --git a/src/types/domain/device-session.types.ts b/src/types/domain/device-session.types.ts index 4c3c7a4..ea4d063 100644 --- a/src/types/domain/device-session.types.ts +++ b/src/types/domain/device-session.types.ts @@ -123,6 +123,8 @@ export interface DeviceCommandResult { /** Each flow of a run. Absent from APIs older than this field. */ flows?: DeviceFlowResult[]; output?: string; + /** A failed run: the app's visible text just after it failed. */ + screen_text?: string[]; success: boolean; } @@ -131,8 +133,13 @@ export interface DeviceScreenElement { /** Bounds, "[left,top][right,bottom]". */ b: string; a11y?: string; + checked?: true; + /** False when disabled; absent when enabled. */ + enabled?: false; + focused?: true; hint?: string; rid?: string; + selected?: true; txt?: string; } @@ -142,6 +149,10 @@ export interface DeviceScreen { /** False when no new frame came in time; this is then the latest. */ fresh: boolean; height: number; + /** Keyboard keys and System UI left out (only when some were); --all shows them. */ + hidden?: { keyboard: number; system: number }; + /** An on-screen keyboard is showing (Android). */ + keyboard_open?: true; keys: Record; width: number; } diff --git a/src/utils/device-run.ts b/src/utils/device-run.ts index b35b838..bc03a5c 100644 --- a/src/utils/device-run.ts +++ b/src/utils/device-run.ts @@ -91,6 +91,20 @@ export function printRunResult( passed: boolean, result: DeviceCommandResult, label: string, +): void { + printOutcome(log, passed, result, label); + // What the app showed after the failure: why an assertion missed. + if (!passed && result.screen_text?.length) { + log(ui.section('On screen after the failure')); + log(ui.branch(result.screen_text.map((t) => `"${t}"`))); + } +} + +function printOutcome( + log: (line: string) => void, + passed: boolean, + result: DeviceCommandResult, + label: string, ): void { const report = parseRunReport(result); if (!report || report.flows.length === 0) { diff --git a/test/unit/device-command.test.ts b/test/unit/device-command.test.ts index 62d4869..85bf4b1 100644 --- a/test/unit/device-command.test.ts +++ b/test/unit/device-command.test.ts @@ -2,6 +2,7 @@ import { expect } from 'chai'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { stripVTControlCharacters } from 'node:util'; import { runCommand } from 'citty'; @@ -9,6 +10,7 @@ import { collectFlagValues, decodeDataUrl, deviceCommand, + formatElement, newLogLines, parseEnvFlags, startNextSteps, @@ -451,6 +453,17 @@ describe('dcd device', () => { expect(out.data).to.deep.equal({ screen }); }); + it('inspect --all asks for the keyboard and System UI too', async () => { + const screen = { width: 390, height: 844, keys: {}, elements: [], fresh: true }; + const calls = stubFetch(() => ({ body: { screen, session: session('a-b-c') } })); + await runJson('inspect', { id: 'a-b-c' }); + await runJson('inspect', { id: 'a-b-c', all: true }); + expect(calls.map((c) => c.url)).to.deep.equal([ + `${API}/device/sessions/a-b-c/inspect`, + `${API}/device/sessions/a-b-c/inspect?all=true`, + ]); + }); + it('screenshot saves the image and reports its path', async () => { const file = path.join(tempDir, 'shot.jpg'); const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xd9]); @@ -731,6 +744,13 @@ describe('dcd device', () => { }); describe('helpers', () => { + it('formatElement shows state that is not the default', () => { + const line = stripVTControlCharacters( + formatElement({ b: '[0,0][1,1]', txt: 'English', selected: true, checked: true, enabled: false }), + ); + expect(line).to.equal('"English" checked selected disabled [0,0][1,1]'); + }); + it('startNextSteps fits the session: check readiness first, install only with no app', () => { const cmds = (s: DeviceSession, hasApp: boolean) => startNextSteps(s, hasApp).map(([, cmd]) => cmd); diff --git a/test/unit/device-run.test.ts b/test/unit/device-run.test.ts index 3c24e4e..5b46647 100644 --- a/test/unit/device-run.test.ts +++ b/test/unit/device-run.test.ts @@ -92,6 +92,17 @@ describe('dcd device run output', () => { expect(out).not.to.include('total_commands_executed'); }); + it("shows a failed run's screen text, so a near miss is plain", () => { + const out = printed(false, { + success: false, + error: 'Assertion is false', + screen_text: ['Star at the centre'], + }); + expect(out).to.include('On screen after the failure'); + expect(out).to.include('"Star at the centre"'); + expect(printed(true, { success: true, screen_text: ['x'] })).not.to.include('On screen'); + }); + it('counts the failures in a run of several flows', () => { const out = printed(false, { success: false,