diff --git a/README.md b/README.md index 2b56acc1..f87d2268 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,9 @@ Build a visual map of your app's screens and the navigation between them, for Ex ### App Intelligence Trace each Expo Router or React Navigation screen through the source files, assets, and packages it depends on, without building or running the app. Choose a project and generate its static iOS and Android graph, then browse entry points by area, fold or highlight dependency groups, and see which dependencies are platform-specific. Ask your agent what a screen depends on, which screens reach a file, or to show one screen's graph. Saved snapshots live under `~/Library/Application Support/mobile-dev/app-intelligence`; project paths and graph contents are excluded from Sentry telemetry. +### Testing Studio +Record deterministic e2e tests against a booted iOS simulator or Android emulator. Choose a project with an existing e2e setup, launch the configured target, and right-click elements in the mirror to add taps, text entry, gestures, and assertions. The generated TypeScript and editable step list stay beside the app; save only when the flow is ready, then run the whole test or play to one step with per-line results. Unsaved recordings are kept in Mobile Dev and stored tests remain in the project's `tests/` folder. You can also ask your coding agent to record a flow: its steps appear live in the same panel for you to review, adjust, and save. + ### Native and JavaScript logs Read iOS unified logs, Android logcat, and Metro console messages together. Follow the foreground app, search and filter logs, and send an error and its stack trace to chat. @@ -190,6 +193,7 @@ Desktop bundle uses Claude's Node runtime, the Cursor plugin uses Node.js from | Metro logs and React Native inspection | An existing local Metro server with a compatible app inspector target. | | Physical iOS CPU and memory | iOS 17.4 or later, a running development-signed app with `get-task-allow`, a mounted developer disk image, and Xcode/LLDB detached. | | Display FPS | Android 12 or later with FrameTimeline support, or a physical iOS device running iOS 17.4 or later. iOS simulators do not support Display FPS. | +| Testing Studio | A booted iOS simulator or Android emulator, plus an existing e2e project with its dependencies installed and a target bundle ID in `e2e.config.*`. | | Profile analysis | A local TraceSift installation, found on `PATH` or through `TRACE_SIFT_COMMAND`. | Simulator tools use the full Xcode installation selected by `xcode-select` (or @@ -218,8 +222,9 @@ Ask your agent to work with your app, for example: - “Record CPU and memory for 30 seconds while I scroll checkout.” - “Compare these two performance recordings.” - “Show me the App Intelligence for this project.” +- “Record an e2e test for the checkout flow.” -Open App Intelligence, Logs, Performance, Storage, or TraceSift. App Intelligence analyzes a project without a device; the other tools inspect the selected device. Use Screenshot to attach +Open App Intelligence, Testing, Logs, Performance, Storage, or TraceSift. App Intelligence analyzes a project without a device; Testing records against a simulator or emulator; the other device tools inspect the selected device. Use Screenshot to attach an image to your next message, or Select to annotate an element or region with an instruction. Physical iOS supports taps and drags; keyboard and hardware-button controls are unavailable. @@ -231,6 +236,7 @@ Detailed guides are available for [logs](https://github.com/callstackincubator/mobile-dev/blob/main/docs/logs.md), [performance](https://github.com/callstackincubator/mobile-dev/blob/main/docs/performance.md), [App Intelligence](https://incubator.callstack.com/codex-mobile-dev-plugin/docs/app-intelligence), +[Testing](https://incubator.callstack.com/codex-mobile-dev-plugin/docs/testing), and [MCP tools](https://github.com/callstackincubator/mobile-dev/blob/main/docs/mcp-tools.md). [Architecture](https://github.com/callstackincubator/mobile-dev/blob/main/docs/architecture.md) explains how the hosts share the server and panel. diff --git a/apps/claude/plugin-host.json b/apps/claude/plugin-host.json index b3c47a60..f943e86f 100644 --- a/apps/claude/plugin-host.json +++ b/apps/claude/plugin-host.json @@ -14,7 +14,7 @@ }, "panelRuntime": "download", "build": "tooling/build.mjs", - "skills": ["skills/mobile-dev", "../../packages/skills/agent-device"], + "skills": ["skills/mobile-dev", "../../packages/skills/mobile-dev-testing", "../../packages/skills/agent-device"], "folders": ["scripts", "hooks"], "readme": "README.md", "maxFileBytes": 5242880 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index feb81a32..46f627ef 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -123,7 +123,7 @@ The README banner uses copy on the left and the large dithered workspace-and-pho - `website/src/components/home/HeroDemo.astro` and `annotation/`: the native dark annotation hero and responsive choreography. - `website/src/components/home/HomeSections.astro`: mobile tools (“A mobile IDE in your ADE”), environment setup, performance recording, storage inspection, and FAQ, in that order. - `website/src/pages/404.astro`: the approved empty-workspace message, requested address, recovery links, and hollow scene. -- `website/src/content/docs/docs/`: Starlight guides for installation, devices, controls, logs, performance, profiles, storage, privacy, and terms. +- `website/src/content/docs/docs/`: Starlight guides for installation, devices, controls, App Intelligence, App Flow, testing, logs, performance, profiles, storage, privacy, and terms. - `website/src/styles/theme.css` and `docs.css`: the dark docs palette, shared fonts, square controls, readable code, hairline sections, and navigation. `website/src/components/starlight/Head.astro` preloads the four fonts used above the fold to avoid a late font swap shifting the page. - `website/src/components/site/dither.ts`, `dither-host.ts`, and `dither.worker.ts`: scene geometry, visibility, sizing, and the worker renderer. - `.github/workflows/website.yml`: GitHub Pages builds and production publishing. `SITE_URL` sets the origin and `SITE_BASE` sets the repository path for navigation, Markdown, assets, and metadata. `website/vercel.json` hosts the same static output for previews. diff --git a/docs/agent-recording.md b/docs/agent-recording.md new file mode 100644 index 00000000..e798741a --- /dev/null +++ b/docs/agent-recording.md @@ -0,0 +1,249 @@ +# Agent-driven recording in the Testing panel + +Status: implemented. The five model tools, `mobile_testing_agent_state` and +`mobile_testing_set_context` are in `packages/core/src/testing-agent.ts`; the panel polls, adopts +and locks the mirror; the skill is `packages/skills/mobile-dev-testing`. Android emulators are supported +through the context's `deviceId` and `platform`; physical devices, Maestro, saving and running are +still out, as below. + +The Testing panel records an e2e test from the user's own taps on the mirror +([docs/testing-panel.md](testing-panel.md)). This adds the other driver: the user tells their +coding agent *"use Mobile Dev and record the test for the important areas on the Explore Details +screen"*, the Testing tab opens, and the steps appear in it one at a time as the agent works. When +the agent stops, the flow is sitting in the panel unsaved and the user adjusts it, saves it and +runs it exactly as if they had tapped it themselves. + +## Decisions + +| Decision | Choice | Why | +|---|---|---| +| Who reasons | The user's own agent, one step per tool call | No model in the server, no second set of credentials, the agent already has the mobile-dev skill and can see the mirror | +| Panel | Live — steps appear as they are recorded | "Opens up the panel for the user" is the point; it also gives the user a Stop button | +| Step kind | Pinned locators; `agent.act`/`agent.assert` only where nothing resolves | Same rule the manual recorder follows: the point of recording is determinism | +| Saving | No. The agent finishes with the steps dirty in the panel | The user adjusts before anything reaches `tests/` | +| Keeping | Yes, as an unsaved recording in the plugin's own storage | Nothing reaches `tests/`, and a flow the user has not opened the panel for yet is not lost — see [docs/testing-panel.md](testing-panel.md#unsaved-recordings) | +| Running | No | There is no file to run, and a run resets the app and takes the device session | +| Prerequisites | Refuse when no project or device is chosen | Project choice stays the user's explicit act, and an OS folder picker must not open inside an agent turn | + +Out of the first pass: physical devices, Maestro, saving, running, and any agent in the +server process. + +## Why the server has to own the session + +The recorder is a singleton in the iframe (`testRecorder`, `packages/ui/src/test-recorder.ts`); +`RecordedStep[]` exists nowhere else. The agent talks to the server over MCP and cannot reach that +object. The only point the two share is the server process, so the agent's recording lives there +and the panel reads it — the same shape as run progress, which the panel already polls because a +tool call is one request and one response. + +```mermaid +flowchart LR + Agent -->|mobile_testing_agent_screen| Srv[AgentRecording in core] + Agent -->|mobile_testing_agent_step| Srv + Srv -->|snapshot + project's @e2e-dev/mobile| Dev[agent-device session] + Srv -->|tap / fill / scroll| Dev + Panel -->|mobile_testing_agent_state, polled| Srv + Panel -->|adopt steps| Rec[TestRecorder] + Dev -.->|frames, unchanged| Mirror[Baguette mirror] +``` + +## The tools + +New file `packages/core/src/testing-agent.ts`, registered beside `registerTestingTools`. + +Model-visible (`_meta.ui.visibility: ["app", "model"]`, the `sharedVisibility` shape already used +in `packages/core/src/app-flow-tools.ts`): + +| Tool | Input | Does | +|---|---|---| +| `mobile_testing_agent_begin` | `{ title, goal }` | Opens the Testing tab, starts one session against the panel's current project, device and bundle id. Refuses, with what is missing, when any of the three is unset. | +| `mobile_testing_agent_screen` | `{}` | The runner's element tree as the agent's eyes: one compact row per element — `ref`, `role`, `name`, `text`, `placeholder`, `value`, `testId`, `enabled`, `selected`, `bounds` — **and the selector the recorder would emit for it**, with its rank and whether it is unique. The agent never writes a locator; it names a `ref`. | +| `mobile_testing_agent_step` | `{ ref?, kind, value?, direction?, instruction? }` | Records one step and performs it. `kind` is a `PinnedStepKind`; `instruction` with no `ref` records an `AgentStep`. Assertions record without touching the device, mirroring the element menu. An acting step answers with `navigated: true` when it replaced the screen — see "Where a flow ends". | +| `mobile_testing_agent_finish` | `{ note? }` | Ends the session with the steps left dirty in the panel. Saves nothing, runs nothing, and keeps the device session for the panel's recorder. | +| `mobile_testing_agent_abort` | `{ reason }` | Drops the session; whatever was recorded stays in the panel. | + +App-only: + +| Tool | Input | Does | +|---|---|---| +| `mobile_testing_agent_state` | `{}` | `{ active, status, revision, title, goal, steps, note, error }`. The panel polls it. | +| `mobile_testing_set_context` | `{ projectPath, udid, bundleId }` | The panel reports its selection whenever it changes. This is the only thing that knows the selected device, and `agent_begin` reads it. | + +The eight existing `mobile_testing_*` tools stay app-only. A narrow, intention-shaped surface is +what keeps `write_file` and `run_test` out of the model's hands, which is what +`packages/core/src/testing-tools.ts:17` already says. + +One session at a time. A second `agent_begin` refuses rather than stomping on the first. + +## Where a flow ends + +The first thing a real user asked for was *"record the test for the important areas on the Explore +screen"*. The recording covered Explore, tapped through to Explore List, and then kept going: +`Explore Cities` asserted, `Spending summary` asserted, the `Search transactions` field filled. +None of that is the Explore screen. + +Nothing had said so. The skill said where a flow *starts* — "the navigation to the screen is part +of the flow" — and a starting point is what the agent read the screen name as. So the skill now +says where one **ends**: the named screen is the *scope*, a control that leaves it may be tapped +and the arrival asserted once, and then the recording stops. A *flow* name ("the checkout flow") +is not a boundary; a screen name is. + +The tool surface helps, because the skill alone is a rule with nothing to apply it to: a screen +read answers with elements, not with screen identity. So an acting step now answers with +**`navigated: true`** when it replaced the screen, and carries the session's own `goal` sentence +back with it, which is the thing the agent has to weigh the new screen against. + +- The signal is the **overlap between the two reads' element identities** — role, testID, name, + text, which are what a test names an element by and are stable across reads, where node ids are + minted per read. A step that *changes* a screen leaves most of it in place; a step that + *navigates* replaces it. Measured on a real app: opening a disclosure kept every element, + switching tabs kept about two fifths of them — the tab bar and the chrome around it — and + tapping into a pushed screen kept about a tenth. The threshold is a third, in the gap between + those. A tab switch not firing is the answer worth having: it is usually the navigation *to* the + screen the recording is for, so a boundary warning there would land exactly as the agent + arrives. +- The extra read costs nothing. An acting step needed one anyway to compare, and that read is the + one the agent's next `mobile_testing_agent_screen` is served from, so the loop still pays one + snapshot per step. +- **It is a hint, not a gate.** Nothing refuses a step, nothing counts navigations, and + `agent_begin` has no scope limit. A checkout and a sign-in cross screens on purpose, and a + recorder that would not let them is worse than one that occasionally records a step too many. + +## Reusing what is there + +- **Selector resolution and emission are already pure** and already in + `packages/shared/src/test-recording.ts`: `resolveSelector`, `stepOffers`, `stepExpression`, + `emitE2eTestLines`. Core imports them. Nothing is reimplemented and nothing is duplicated, so an + agent-recorded step and a hand-recorded one are the same `RecordedStep` and emit the same line. + The one change needed: resolution is reached today through `hitLadder`, which starts from a + point. The agent starts from an element, so the element-shaped entry point is factored out and + the point path calls it. +- **Reading** goes through the existing `TestingDevices.read` — `capture.snapshot` projected by the + project's own `@e2e-dev/mobile`, on the `mobile-dev-recorder` session. +- **Acting** is new: `TestingDevices.perform({ ref, kind, value })` on that same session, through + agent-device's own tap/fill/scroll. Not the panel's baguette input path — the server cannot reach + it, and agent-device taps the element the way the runner will. The mirror still shows every + change, because the stream and the snapshot already hold the simulator at once. + This means each projected node must keep its agent-device node beside it so a `ref` maps back. +- **The device is released** on the paths that already release it. An agent session is one more + holder of the same `mobile-dev-recorder` session, not a second one. + +## The panel + +`TestingPanel` polls `mobile_testing_agent_state` — 400 ms while a session is active, a slow tick +otherwise so a session the agent started is noticed. + +- On a revision bump, `TestRecorder.adopt(steps)` replaces the step list. The preview's existing + `changedLines` animation flashes the new lines, so a step the agent records reads the same as a + step the user records. +- `recordingUdid` is set for the agent's device, so the step list, the emitter, editing, + duplication and Save all work unchanged. This is deliberately not a second state machine. +- While a session is active: a banner reads **Agent is recording — ** with a **Stop** button + that calls `agent_abort`; left-click drive and the right-click element menu on the mirror are + disabled, because the agent and the user must not both be acting on one screen. +- On `finished`: the banner clears, the steps stay dirty, and the panel says what was recorded and + that nothing has been saved. From here it is an ordinary unsaved recording. The server has kept the + flow by then and `agent_state` carries its `draftId`, which the panel adopts so its own autosave + replaces that draft rather than leaving a second copy of one recording in the list. + +## The skill + +A new skill, `packages/skills/mobile-dev-testing/`, shipped the way `packages/skills/README.md` +describes: all three plugins for Codex and Cursor, and `apps/claude/skills` for Claude Code. + +It covers: +- **When**: the user asks to record, write or capture an e2e test for a screen or a flow. +- **Prerequisites**: the Testing tab, a chosen project, a booted iOS simulator with the app + launched. Missing any, say which and ask — never open the folder picker on the user's behalf. +- **The loop**: `agent_begin` → `agent_screen` → choose a `ref` → `agent_step` → re-read → … → + `agent_finish`. +- **What to record**: the test starts from `app.open()`, so the navigation to the target screen is + part of the flow and is recorded. Then the screen's landmarks as assertions, its one or two real + affordances exercised, and the result asserted. Keep a flow under about fifteen steps; a long + flow is two tests. +- **Pinned first.** Use `agent.act`/`agent.assert` only for a `ref` the screen read reports as + having no unique selector. +- **Never save and never run.** Finish and tell the user the flow is in the panel to adjust. + +## Telemetry + +Per [AGENTS.md](../AGENTS.md): count `ui.testing.agent_session_start`, `agent_step_adopted` and +`agent_session_abort`; time the server's screen read and the panel's adopt-to-render. Failures go +through `captureServerError` / `captureUiError`. The goal sentence and element names are user +content and are **not** sent — only counts and durations. Document the new names in +[docs/telemetry.md](telemetry.md). + +## Build order + +1. Element-shaped selector resolution factored out of the point path, plus + `TestingDevices.perform` and the ref↔node mapping. Tests in + `packages/shared/test/test-recording.test.ts` and `packages/core/test/testing-device.test.ts`. +2. `testing-agent.ts`: the session, the five model tools, `mobile_testing_set_context`, + `mobile_testing_agent_state`. Provable with the tools alone, before any UI. + `packages/core/test/testing-agent.test.ts`. +3. The panel: the poll, `TestRecorder.adopt`, the banner, the input lockout and the handover. + `packages/ui/test/test-recorder.test.ts`, `packages/core/test/testing-panel-ui.test.ts`, and + refreshed visuals via `npm run test:visual:docker -- --update-snapshots`. +4. The skill, `docs/telemetry.md`, and this document's status line. +5. Prove it against `/Users/hurali/Desktop/callstack/test-projects/expoapp57`: *"record the test for + the important areas on Explore Details"*, which should navigate Explore → Explore List → Paris, + assert the heatmap and its description, fill "Search reviews" and assert the filtered review row + — roughly `tests/city-review-search.e2e.ts`, which a human recorded by hand. + +## What the real device changed + +Five things in the plan did not survive contact with a booted iPhone and the project's own runner, +and each was adjusted rather than forced. + +- **The ancestor chain stops below the screen's own root.** The point path never reaches a + full-screen root because `hitLadder` drops one; an element-shaped chain walks `parentId` up and + reached it on every unnameable node, so a scroll bar resolved to `screen.getByText("expoapp57")` + and the read reported it *unique*. `resolveComponentSelector` now takes the viewport and applies + the same rule. `anchorSelector` had the same latent bug — the root is the nearest element to any + point — and now takes it too, so a coordinate step no longer waits for the application to exist. +- **Every action settles.** The step after one is resolved against the screen it produced, and + agent-device refuses an action on a ref it captured mid-transition: *"had no confirmed resting + position when the transition window ran out"*. The wait belongs where the transition is, so + `perform` passes `settle: true` rather than leaving the next read to guess. +- **A viewport swipe is agent-device's `scroll`, not a trajectory of ours.** Deriving one from the + snapshot's viewport is refused with *"Gesture trajectory does not fit inside the viewport"* the + moment the on-screen keyboard shrinks the window the gesture runtime measures. The runner + performs `screen.swipe` as `client.interactions.scroll({ direction })` + (`@e2e-dev/mobile/dist/surface.js`), so the recorder does the same call with the same direction. + It also stopped carrying the coordinate-tap TODO, which was never true of it, while keeping the + anchor gate, which is. +- **Off-screen elements stay in the read, flagged `offScreen`.** The first version dropped them as + the mirror does, and the flow being recorded was a search filter whose *result* is a row below + the fold. `getByText` resolves it either way, so dropping it hid exactly what the test was being + written about. +- **agent-device's refs keep their `@`.** `@e2e-dev/mobile`'s projection strips the sigil for its + own use and `interactions.click({ ref: "e20" })` then parses the ref as a selector and refuses + it. It is put back at the one boundary that hands a ref to agent-device. + +Two smaller departures from the text above: the panel **releases the device** when an agent +recording ends, exactly as the Stop button does, because a leaked `mobile-dev-recorder` session is +the failure `docs/testing-panel.md` spends a section on and Record re-attaches in a tenth of a +second; and the panel **restores the last project when the host becomes available** rather than +only when the tab is opened, so an agent recording does not require the user to open the tab first +only to have it reopen on the folder they already chose. + +### Proved against expoapp57 + +Driving the five tools from outside the panel, one step per call, against a booted iPhone 17 Pro +produced `tests/agent-explore-details.e2e.ts` in that project — Explore → Explore List → Paris, the +heatmap and its description asserted, "Search reviews" filled, the value and the featured reviewer +asserted, and back — and the project's own `e2e run` passes it. The one assertion the hand-recorded +`tests/city-review-search.e2e.ts` has and this does not is the filtered review row: with the +keyboard up it is past the end of the scroll content and never enters the snapshot, so the agent +could not read the text it would have asserted. It asserts what it can see, which is the rule the +skill states. + +## Risks + +- **Ref staleness.** A step must act on the element it resolved. Every action invalidates the tree, + as the manual recorder already assumes, so the agent re-reads after each one and a `ref` from an + older read is refused rather than acted on. +- **Cost.** A screen read is about 0.1 s and a flow is ten to twenty of them. Acceptable; the + manual recorder pays the same per tap. +- **Two drivers, one screen.** Solved by locking the mirror's input while a session is active, not + by hoping. diff --git a/docs/telemetry.md b/docs/telemetry.md index 60b29c05..37179a16 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -537,6 +537,106 @@ it does not confirm user submission or a model response. Notes remain until the user clears them. Message/context construction timings, surface attribution, and content scrubbing are preserved; no note text or selection evidence enters telemetry. +The Testing panel's recorder measures `ui.testing.tree_read`, the read that resolves a tap into a +selector, and counts `ui.testing.record_start`/`record_stop`, `ui.testing.step_tap`, +`ui.testing.step_assert`, `ui.testing.step_unresolved`, `ui.testing.hover_walk`, +`ui.testing.menu_open`. Editing a recorded step counts `ui.testing.step_duplicate`, +`ui.testing.step_reorder`, and `ui.testing.step_recommand_action`/`ui.testing.step_recommand_assert` +when a step is pointed at another command — the same action/assertion split the recording counters +use, so a flow's shape stays readable whether a step was recorded or edited into place. They are +counts of an edit, carrying no selector, element name, command text or position. Pressing Record +on a flow that already has steps counts `ui.testing.record_resume` or `ui.testing.record_new` +alongside `ui.testing.record_start`, and only where there was a question to answer — it says +whether that choice is carrying its weight, and carries nothing about the flow itself. +Recording an agentic step counts `ui.testing.step_agent_act`, `ui.testing.step_agent_assert` or +`ui.testing.step_agent_wait`, +kept apart from `ui.testing.step_tap`/`step_assert` because the two answer different questions: +whether the recorder can pin a step, and whether a user reached for the agent instead. They are +counts only — the instruction the user wrote is free text about their own app, so it is never +sent, and neither is the `vision` mode chosen for an `agent.assert`. `ui.testing.agent_params` +gauges how many params an `act` step was recorded with, which is what says whether the editor is +reached for at all; it is a count of rows, never a key, a value or the text they were typed into. +A recorded `swipe` counts `ui.testing.step_action` like every other non-tap pinned command, so the +action/assertion split stays the boundary it was; the direction and the gesture's coordinates are +not sent. +`ui.testing.step_assert` now covers every assertion the menu can record, not only `toBeVisible` +and `toHaveText`; it is the same boundary — a step recorded that reads the screen rather than +acting on it — so counts stay comparable across the release. Selectors, element names and the +recorded source are not sent; these are counts only. **The timing's +meaning changed when the recorder moved to the test runner's own element tree**: it now measures an +agent-device snapshot of the device projected by the project's `@e2e-dev/mobile` (around a tenth of +a second against a booted iPhone 17 Pro), where it previously measured a Baguette `describe-ui` +loopback of tens of milliseconds. It is the same boundary — the wait a tap pays before its step is +named — so the measurement stays comparable in purpose, but not in magnitude; read a step across +that release as a change of read path, not a regression. The recorder caches the tree for 1.5 s and +invalidates it on every input, so a warm hover or wheel walk is answered locally and not measured. +`ui.testing.tree_chrome_only` is gone with the Baguette path, since the runner's snapshot is rooted +at the application and cannot report only the simulator's chrome; `ui.testing.tree_empty` replaces +it and counts a tree with no elements at all. Handled failures of the read and of releasing the +device go through `captureServerError` as `testing.tool` and `testing.release`. No element names, +labels, placeholders, testIDs, roles, bounds, bundle ids, project paths, device identifiers or +emitted test source are attached to any of these. + +An **agent recording** — the user's own coding agent writing the flow through +`mobile_testing_agent_*`, see [docs/agent-recording.md](agent-recording.md) — is measured on both +sides of the one boundary it crosses. In the server, `testing.agent.screen.duration` times the +element read an agent asks for before each step, `testing.agent.screen.samples` counts those reads +by outcome, and `testing.agent.screen.elements` gauges how big the screen it answered with was. +That read is the only thing in an agent recording that costs anything — a flow is ten to twenty of +them — and the element count is a size, never content. In the panel, +`ui.testing.agent_session_start` counts a session the panel adopted, +`ui.testing.agent_step_adopted` counts each step as it arrives, `ui.testing.agent_session_abort` +counts the user pressing Stop on the banner, and `ui.testing.agent_adopt` times the adopt that +makes a step appear. The goal sentence, the test's title, the instruction of an agent step and +every element name the read answered with are the user's own content about their own app and are +**not** sent; these are counts, durations and sizes only. Handled failures of the agent tools go +through `captureServerError` as `testing.agent`, beside `testing.tool` and `testing.release`. + +Running a recorded test measures `ui.testing.run`, which brackets the panel's whole call — the +runner spawned, the project resolved and the report read — and counts `ui.testing.run_passed` or +`ui.testing.run_failed` from the verdict. It is deliberately not the duration shown beside that +verdict: that one is the runner's own number from the report, and the two answer different +questions. Play until here, the run a line of the preview offers, counts `ui.testing.run_until` +as well, so a flow tried a step at a time is not read as a flow run end to end; it is counted +beside the verdict, not instead of it. The test's title, its path, the project path, the steps and +the runner's output are the user's own content and are **not** sent. + +Unsaved recordings — the flows the panel keeps so one survives the panel document being rebuilt — +measure `ui.testing.draft_save`, the autosave write, and count `ui.testing.draft_restore` when one +is reopened into the step list and `ui.testing.draft_delete` when the user throws one away. The +measurement brackets the panel's call and the list refresh that follows it, so it says what the +keep costs the user between one edit and the next; the keep the panel makes as it is torn down is +not measured, because the document it would be attributed to is going away. The title, the file name, the project path, the +steps and everything they name are the user's own content and are **not** sent; these are counts +and a duration. An autosave that fails is silent on screen and goes through `captureUiError` as +`testing.draft_save`; the server's own keep of a finished agent recording goes through +`captureServerError` as `testing.agent.draft`. Deleting one of the project's own stored tests +counts `ui.testing.test_delete`, beside `ui.testing.draft_delete`: they are the two halves of the +panel's list and answer different questions — a recording thrown away before it was ever saved, and +a test removed from the repository. The file name, its path and the project path are not sent, and +listing the stored tests is not measured: it is a folder read the panel does on open, not a wait +the user sits through. +Selecting a stored test counts `ui.testing.test_preview` and measures that same name from the +read request through its response. A failed read goes through `captureUiError` as +`testing.test_preview`. The test source and path are not sent. +Editing a project test counts `ui.testing.test_edit` and measures `ui.testing.test_stage` around +the temporary write. Save counts and measures `ui.testing.test_save`; Undo counts and measures +`ui.testing.test_undo`. Handled staging, save and undo failures go through `captureUiError` as +`testing.test_stage`, `testing.test_save` and `testing.test_undo`. The source, path, and command are not sent. + +`ui.testing.record_prepare` times the warm tree read Record now does before the mirror can resolve +an element, which is the wait the `preparing...` overlay is explaining. It is deliberately **not** +folded into `ui.testing.tree_read`: that one covers every read, warm and cold, during a recording, +while this one answers "how long is Record unusable for" — the number the overlay exists because +of. A read that fails still records the timing, because the wait happened; it carries no selector, +element name, bundle id or project path. + +The Testing panel records on Android emulators as well as iOS simulators, so `ui.testing.tree_read`, +`ui.testing.record_prepare` and the `testing.agent.screen.*` metrics now carry a `platform` +attribute (`ios` or `android`; two values, no device ids). One series hid that waiting on an +emulator is a different number from waiting on a simulator. Names, units and boundaries are +unchanged, and no span or per-frame event was added. + Since 0.1.99, `ui.logs.foreground_change` counts automatic app-filter changes applied while Logs is active, including selected-device and process-lifetime changes. From 0.1.103, these changes update the visible query and local process identity mapping instead of restarting a scoped collector. `ui.logs.app_identity` measures batch insertion with native PID-to-app joins and discovery updates in milliseconds, using bounded aggregate windows. Hidden or closed log views defer row processing until reopened. Shared `ui.device_apps.discovery` timing and handled discovery-error coverage remain in place, including Android's foreground PID lookup; native session operations retain MCP tracing, Node runtime coverage, and `logs.ios.parse` timings. No app IDs, PIDs, package names, device IDs, or query text are attached to these measurements. Native helper telemetry and symbols are unchanged. Since 0.1.99, `ui.logs.query_parse` measures query compilation in milliseconds once per edit, including automatic app-clause edits from 0.1.103. Existing `ui.logs.filter`, buffered/filtered row gauges, and search counts cover keyword filtering and visible age refreshes; filtering time still covers snapshot derivation and grouping. Query text, field values, regex patterns, and validation messages remain local. Age refresh timers stop when Logs closes, unmounts, or the document becomes hidden. diff --git a/docs/testing-panel.md b/docs/testing-panel.md new file mode 100644 index 00000000..5001915f --- /dev/null +++ b/docs/testing-panel.md @@ -0,0 +1,872 @@ +# Testing panel — record e2e tests from the mirror + +Status: Android emulators are supported alongside iOS simulators — the device is identified by +`deviceId` plus an explicit `platform`, and the e2e run is pinned with `--target`. The Android path +is not yet verified on a real emulator, including the pixel space below. Steps 1-6 of the build +order are implemented — the tab, the project picker, launching the +app, recording through the mirror's element menu with selector resolution and a live preview, +saving the test into the project, and running it with per-line marks. Typing and swipes are +recorded from the menu rather than from the gesture, which is step 5's own answer; a drag on the +mirror still records nothing (step 7), and the Maestro emitter (step 8) is still design only. +A test can also be recorded by the user's own coding agent, which drives the same recorder from +the server — see [docs/agent-recording.md](agent-recording.md). + +A new workspace tool, **Testing**, that turns the existing simulator mirror into a +Maestro-Studio-style recorder: pick a project, launch its app, tap the mirror, and watch the test +source grow one step per interaction. The file is written to the project when the user clicks Save. + +First pass targets the **iOS simulator** (and now the **Android emulator**) and the **e2e** runner (`@e2e-dev/mobile`). +Maestro YAML is a second emitter behind the same recorder. + +## Scope of the first pass + +| Decision | Choice | +|---|---| +| Platform | iOS simulator (Baguette stream path) and Android emulator (`serve-emu`/scrcpy); physical devices excluded | +| Tap → code | Resolve a selector from the accessibility tree; fall back to coordinates | +| Output | Live **preview** in the panel; written to the project's `tests/.e2e.ts` only on **Save** | +| Emitter | e2e TypeScript now; Maestro YAML later, same recorder | + +Physical iOS (`ios-mirror`) and physical Android are out. Android emulators slot in behind the same +interfaces: the recorder reads the same agent-device snapshot with `{ platform: "android", serial }`. + +## What already exists + +Almost every primitive is in place; the panel is mostly wiring. + +| Need | Existing piece | +|---|---| +| Mirror frames | `packages/core/src/stream-sessions.ts:78` → MCP `resources/read`, drawn on canvas in `packages/ui/src/simulator-panel.ts` | +| Pointer capture over the mirror | canvas `pointerdown/move/up` at `packages/ui/src/simulator-panel.ts:855-892` | +| Touch injection | `mobile_stream_input` (`packages/core/src/panel-tools.ts:343-390`) → `stream-input.ts` batching | +| Accessibility tree | `baguette.describeUi(udid)` (`packages/core/src/baguette.ts:118`), exposed as `mobile_describe_ui` (`panel-tools.ts:292`) | +| Hit-testing a point | `componentAt` / `componentsAt` (`packages/shared/src/screen-annotations.ts:89,120`), already driving hover/select in the mirror | +| Tab registration | `toolNames` in `packages/ui/src/workspace-tools.ts:11`, label/icon map in `packages/mcp-app/src/components/workspace.tsx:27` | +| Maestro replay (later) | agent-device script-replay runs compatible Maestro YAML (`packages/runtimes/agent-device/tools.json`) | + +Two things do **not** exist: + +1. **No path writes into the user's project.** Every `writeFile` in the server targets its own + state (recordings, tracesift, clipboard temp files). The server does not know a project root. +2. **No codegen of any kind.** + +## Unsaved recordings + +A flow lives in the recorder, which lives in the panel document, and the host rebuilds that +document whenever the user looks at another tool or another chat. Until Save writes a file there +was nothing else holding it, so a recording the user had not saved — or one their agent made while +they were reading its answer in the chat — was simply gone when they came back. Both are the +ordinary way the panel is used, which made it the ordinary way work was lost. + +So the panel keeps what it is holding, and lists what it has kept. + +| Decision | Choice | Why | +|---|---|---| +| Where | `~/Library/Application Support/mobile-dev/testing/drafts/.json`, beside `projects.json` | The plugin's own state. Nothing reaches the user's repository until they press Save, which is the whole promise of the Save button | +| What | The `RecordedStep[]`, not the emitted source | A flow comes back **editable**. The generated file is an output and nothing reads it back | +| When | Autosaved 700 ms after the last change, and once more as the panel is torn down | One write per edit burst, and the teardown flush covers the switch that used to lose the flow | +| Who | The panel for its own recording; the server for an agent's, at `agent_finish`/`agent_abort` | The panel may never have been looking. The server publishes the draft's id in `agent_state`, and the panel adopts it rather than keeping a second copy | +| Scope | Listed per project | A flow names elements of one app; another project's selectors have never existed in it | +| Until when | Deleted when the flow is written into the project, or when the user deletes it | The draft stood in for a file that now exists | +| How many | 30, oldest dropped | A safety net, not an archive | + +A draft belongs to a **flow**, not to the panel. The recorder mints a `flowId` whenever the step +list stops being the same recording — Record on a flow the user did not choose to resume, Clear, a +kept recording reopened, an agent session beginning — and the panel keeps the draft pinned to that +id. `revision` cannot stand in for it: it moves for an edit and for a brand new flow alike, and +when the panel went by revision alone, recording a second test wrote it straight over the first +one's file. Two recordings, one row in the list. + +The control is a popover beside the project picker, shown only once there is something in it, and +it holds **two groups**: + +| Group | Holds | Starts | +|---|---|---| +| **Not stored** | The drafts: a kept flow with its step count, how long ago it was touched and whether the user or an agent recorded it. **New recording** is at its foot | Expanded — it is what is being worked on | +| **In the project** | The `.e2e.ts` files in the project's own `tests/` folder, newest first | Collapsed — it is the archive | + +The second group exists because deleting the draft on save had an effect nobody asked for: the +test the user had just saved **left the list**. There was nowhere for it to go. Now it moves from +the top group into the bottom one, which is also the honest description of what Save did. The +bottom group is **read from the folder** rather than remembered, so a test written by hand, by an +agent or in another checkout is listed beside the recorded ones, and a test deleted outside the +panel stops being listed. + +Opening a draft keeps the flow on screen first, so swapping between two loses neither, and it does +**not** start recording — the user opened a flow to look at it, and a stray tap on the mirror must +not append to it. A stored test can be selected to preview its source and awaited calls. The +source calls can be reordered, duplicated or changed to compatible commands. Each edit updates a +temporary working copy, not the project file. Save writes the working copy to the original path +and removes it; Undo removes the working copy and reloads the original. Save refuses if the +original changed on disk after the copy was opened. Each playable call keeps a visible Play icon +beside its source line. Play runs a temporary sibling of the source, +so neither a full run nor Play until here saves pending edits. The current recording stays intact +and can be reopened from Not stored. + +Both deletes **ask first**, in a popover hanging off the trash icon. A draft's steps cannot be +replayed from anywhere else, and a stored test is a file in the user's repository that nothing here +can put back — and both icons sit beside the row the user was reaching for. The question hangs off +the icon rather than replacing the row so the row it is about stays readable underneath it, and so +the question has room to name what will go and say that it cannot be undone, which a row the width +of a file name does not. + +Ten app-only tools carry it: `mobile_testing_drafts`, `mobile_testing_draft_read`, +`mobile_testing_draft_save` and `mobile_testing_draft_delete` over `TestDraftStore` +(`packages/core/src/testing-drafts.ts`), and `mobile_testing_tests`, +`mobile_testing_read_file`, `mobile_testing_stage_file`, `mobile_testing_save_staged_file`, +`mobile_testing_discard_staged_file` and `mobile_testing_delete_file` over the project's own `tests/` folder +(`packages/core/src/testing-projects.ts`) — Save and delete are annotated destructive, so the host +asks as well as the panel. All ten are in +`packages/core/src/testing-tools.ts`. The step schema they validate against is +`packages/shared/src/test-drafts.ts`, which is the one place a `RecordedStep` is described as data +rather than as a type. A stored test's path goes through the same `resolveTestPath` boundary the +write does, so a delete cannot climb out of `tests/`. + +## Architecture + +```mermaid +flowchart LR + Canvas[Mirror canvas] -->|pointerdown / contextmenu| Rec[TestingPanel recorder] + Rec -->|mobile_testing_read_tree| Tree[agent-device snapshot, projected by @e2e-dev/mobile] + Tree --> Hit[hit ladder] + Hit --> Step[RecordedStep] + Rec -->|mobile_stream_input| Device[Simulator] + Step --> Emit[emitE2eTestLines] + Emit -->|mobile_testing_write_file| File[project tests/*.e2e.ts] + Emit --> Spans[per-step line spans] + File -->|mobile_testing_run_test| Run[e2e run --reporter json] + Run --> Spans + Spans --> View[Preview, marked per line] +``` + +### Where the mirror sits + +In the Testing panel the mirror is on the **left** and the preview and step list on the right; +every other tool keeps the simulator on the right. Recording is done by tapping the mirror and read +back in the preview beside it, so the mirror leads in the same direction the panel is read. It is +one flag in `packages/mcp-app/src/components/workspace.tsx` that reverses the resizable group's +children, and only when the workspace is side by side — stacked, "left" is the top panel, which the +simulator already is. The hover outline is unaffected: `TestRecorderOverlay` positions it from the +annotation store's viewport, which is measured **inside the stage**, so moving the stage within the +workspace does not move the rectangle, and the stage's own `ResizeObserver` re-measures when the +swap changes its width. + +#### The mirror has to be there when the panel opens + +The panel is useless without a device: with no mirror it shows "Boot a simulator to record against +it." while the user is looking at a booted simulator. That is what it did, and the cause was not in +this panel at all. The workspace's automatic layout was decided **once**, from the first device +list — and the opening tool *starts* the backend, so that first list routinely comes back before +anything has been found. It resolved to `none`, both device panels stayed hidden, the platform +selector read "Platforms", and nothing ever asked again; discovery found the simulator a moment +later and the panels selected it, but the layout was already settled. Picking the platform by hand +was the only way to see a device that had been there all along. + +`WorkspaceDevices.settleLayout` (`packages/ui/src/workspace-devices.ts`) is the second look: a +panel's selection changing — which is what discovery does — finishes a layout that is still +unresolved. It only ever turns `none` into a platform. A layout that resolved to something, and +any layout the user chose themselves, are left exactly as they are. + +#### The three panes + +| Pane | Default | Why | +|---|---|---| +| Mirror | 25% of the workspace | It is tapped, not read. A phone at a quarter of a wide workspace is still a usable target | +| Preview | 45% | A generated test file is the thing being read, and a wrapped line of `screen.getByRole(…)` is unreadable | +| Steps | 30% | A list of short labels, which is what is left | + +All three are **draggable**: the mirror against the tools through the workspace's own handle, and +the preview against the steps through a handle of their own +(`testing-preview-resizable`/`testing-steps-resizable` in +`packages/ui/src/components/testing-view.tsx`). The split that suits a flow changes while the flow +is being written — a long file to read, a long list to reorder — so the numbers above are defaults +rather than a layout. The preview and the steps are 60/40 **of the tools pane**, which is what +makes them 45% and 30% beside a mirror with 25%; the panel's own padding and the gaps between the +panes come off the two that are read, so measured against the whole window they land a couple of +points under that. + +**The row is asked for by this panel, not inherited from `split`.** `split` is `wide && fullscreen` +— true only in the fullscreen workspace — and everywhere else the workspace group was vertical: +the mirror kept the 60% default meant for a tool stacked underneath it, and the preview and the +steps were left sharing the 40% of height below, which at a narrow tools pane stacked them too. +That is how the three panes came out at roughly 60/22/16 instead of 25/45/30. `testingSplit` in +`packages/mcp-app/src/components/workspace.tsx` now asks for a horizontal workspace wherever the +window is wide enough for one, and the panel's own group has no stacked form at all. Under that +width the workspace stacks as it always did, with the mirror on top. `tooling/hosts/hosts.spec.ts` +holds the three shares against both the fullscreen workspace and the panel beside the chat. + +### 1. The tab + +- Add `"testing"` to `toolNames` (`packages/ui/src/workspace-tools.ts`). +- `TestingPanel` controller in `packages/ui/src/testing-panel.ts`, same `subscribe`/`getSnapshot` + store pattern as `LogsPanel`. +- `packages/ui/src/components/testing-view.tsx` renders the step list, the project picker, the + record/stop control and the generated source. +- Entry in the label/icon map in `workspace.tsx`. + +The view sits where the other tool views sit — beside the mirror, which `DeviceWorkspace` +already renders. No new mirror. + +### 2. Project selection and launch + +The panel has no notion of a project today. Add: + +- **The user always picks the project explicitly** — the panel never infers it from the server's + cwd. A folder picker; the choice is persisted so it is one click on the next session. Validate it by + reading `e2e.config.ts` (or `package.json` depending on `e2e`). Persist the last choice in + panel state. +- Read the chosen `e2e.config.ts` to learn the **target** and the **bundle id**, so the recorder + launches the same app the generated test will (`app: { bundleId: … }`). Parsing TypeScript config + is the awkward part — prefer shelling out to the project's own `npx e2e config --json` if the CLI + exposes it; otherwise fall back to asking the user for the bundle id once and caching it. +- Launch via the existing device/app tooling (`DeviceAppsStore`), not a new path. + +### 3. Recording a tap + +The ordering matters, and it is the one genuinely tricky part: + +1. `pointerdown` on the canvas. **Before** forwarding input, the recorder needs the hierarchy of + the screen as it is *now* — after the tap it may be gone. +2. Keep a **cached tree**, refreshed on screen-settle (the mirror already knows when frames stop + changing) and invalidated on every injected input. On `pointerdown`, hit-test against the cache; + if the cache is stale, issue `mobile_describe_ui` and hold the input for one round trip (tens of + ms on loopback — acceptable; a dropped tap is not). +3. Forward the touch to the device exactly as today, so the app behaves normally while recording. +4. `componentAt(point)` gives a `ScreenComponent { role, identifier, label, value, bounds, react }`. + +While recording, `pointermove` runs the same resolution against the same cached tree and draws an +outline over the mirror with the step it would record (`packages/ui/src/components/test-recorder-overlay.tsx`). +It is a preview, not a second code path: what is outlined is what a tap records. The outline is +cleared whenever the cache is invalidated, because the screen it described is gone. + +**Record shows a `preparing...` overlay until selection actually works.** Pressing Record arms the +recorder at once, but nothing on the mirror can be *named* until the runner's element tree has been +read — attaching to the app and projecting the tree takes a second or two — so the panel used to +say "Stop" while the first few seconds of pointing did nothing, with nothing on screen to say why. +That reads as a broken recorder rather than as a wait. `start` now warms the tree itself and holds +`preparing` until that read lands, and the overlay covers the **canvas** (not the whole stage), so +it says where the wait is. It lets clicks through: the mirror still drives the app while the tree +is being read. A read that fails drops the flag rather than leaving the overlay up — the overlay +must never outlive the thing it is explaining — and the first hover retries as it always did. The +wait has its own timing, `ui.testing.record_prepare`, because "how long is Record unusable for" is +a different question from `ui.testing.tree_read`, which also covers the warm reads during a +recording. + +**Pointing at a step tints the lines it emits**, and the tint goes with the pointer. What made +*leaving* a step light its lines up again — and then fade them out over more than a second — was +not the clearing at all: the highlight used to be a `background` plus `animation: none`, and the +lines a step added keep `is-new` for as long as the file does not change. Suppressing the arrival +animation while a line was highlighted **cancelled** it, and letting it back on mouse-leave +**restarted it from the beginning**. The highlight is painted as a `::before` layer now, so the +two never touch the same property and there is nothing to restart. The row clears the highlight +when the pointer leaves it, guarded so that a row already left cannot take it from the row the +pointer is on: the browser dispatches the leave of one and the enter of the next together, and the +order is not the one the eye follows. + +### 4. Selector resolution + +Priority, first match wins: + +| Rank | Source | Emitted | +|---|---|---| +| 1 | `testId` | `screen.getByTestId("Submit")` | +| 2 | `role` + accessible name | `screen.getByRole("tab", "Explore")` | +| 3 | `placeholder` | `screen.getByPlaceholder("Search transactions")` | +| 4 | name or text | `screen.getByText("Explore Cities")` | +| 5 | accessible name | `screen.getByLabel("Forward, File-based routing")` | +| 6 | nothing resolves | `screen.tapAt(x, y)`, preceded by a wait, + a `// TODO: add a testID` comment | + +These are the runner's own query kinds, counted the way `locate.js` counts them, over the runner's +own tree — see "Reading the runner's tree" below. In particular: + +- **A role query needs a name.** `getByRole(role)` alone matches every element of that role, so a + role with no accessible name is not a candidate at all. A search field whose only text is its + placeholder has no name, which is why rank 3 exists: iOS accessibility offered the placeholder + *as* the name, the recorder emitted `getByRole("textbox", "Search transactions")`, and it + matched nothing. +- **The role is never translated.** It is whatever the engine's projection said, passed through. + The one thing the recorder knows about roles of its own is which ones `getByRole` *accepts* — + `e2e`'s `Role` union is closed, and the engine deliberately keeps an element type it has no role + for as the type itself (`other`, `application`). Those fall through to ranks 4 and 5 rather than + reaching the file as `getByRole("other", …)`, which would not compile. +- **`getByText` and `getByLabel` answer with the innermost match only.** A device tree echoes text + upwards — iOS reports a React Native `Text` host view and the `StaticText` inside it with the + same label — so counting both would flag as ambiguous nearly every piece of text on screen, + while the runner resolves it uniquely. +- **A role query skips a hidden node**, as the runner does, so an off-screen twin is not a second + match. +- Text is compared with whitespace collapsed, because the runner normalizes and an accessibility + label carries the rendered line breaks. + +`screen.tapAt` is the only step with nothing to wait on: every locator action waits up to the +action timeout for its element, but a coordinate is always "ready", so a recorded coordinate tap +fires into the splash screen and misses. A point step therefore carries an **anchor** — the +nearest element on the same screen that resolves uniquely — and the emitter puts +`await expect().toBeVisible()` in front of the tap as the gate the tap cannot express. + +An **assertion** that resolves to a point is refused outright rather than recorded: there is +nothing for `toBeVisible` to name, so the step would emit only comments. + +The ranks are tried over the **hit ladder** — every element the point sits inside, ordered by area, +innermost first — not over one element. The innermost element often carries nothing addressable +while something just outside it holds the name or the testID, and the wheel walks the outline out +and back in when the element under the pointer is not the one the test should name. + +An icon-only control with no name of its own takes the **one** piece of text inside it; two and it +stays unnamed and the ladder moves outwards, rather than guessing. + +**The simulator's chrome is not in the tree.** The clock, the Dynamic Island and the battery used +to arrive in the same tree as the app's own elements, and were chosen as steps and as waits — a +recorded flow once opened with +`await expect(screen.getByLabel("Dynamic Island, Empty")).toBeVisible()`. The runner's snapshot is +rooted at the application, so there is nothing to filter: `treeHealth` is down to `ok` and +`no-accessibility`, and the recorder refuses a step only when the tree is empty. + +**Depth beats rank**: the innermost element that can be named at all wins, and the ranks above +only order the candidates *within* one element. An enclosing element is reached by pointing at it +or by walking out, never by quietly outranking what is under the pointer. + +**Walking out.** The outline is at a level on the ladder, and the wheel over the mirror moves it: +out to the enclosing element, back in again, re-resolved from the cached tree at no cost. The label +reads `3/7`. Pointing at a different element starts again from the innermost. The recorded step is +whatever the outline shows, so what you see is what is written. + +**Roles are passed through, never translated.** The engine's projection already mapped the +platform's element type onto the runner's role vocabulary, and it is the only thing that gets a +say: a second mapping here is what put `getByRole("link", …)` and `getByRole("rcttext", …)` into +saved files. The recorder's only knowledge of roles is which ones `getByRole` accepts; see +"Selector resolution" above. + +Uniqueness check: run the candidate selector back over the same tree. If it matches more than one +node, fall to the next rank, and if all ranks are ambiguous, emit the best one with an index and +flag the step in the UI. The step list should show the chosen selector and let the user override it +from the other candidates — that override is what makes recorded tests maintainable. + +iOS bounds are device points, matching the mirror's own coordinate space after the canvas scale +factor is applied; no normalization needed on this path. Android's mirror and the snapshot are +both expected to be in device pixels, so no conversion is added; confirm by hovering the mirror on +a real emulator. + +### 5. Gestures beyond tap + +Typing, long press and scroll-to are recorded, but **not by watching the gesture** — by asking for +them on the element, through the menu in the next section. A recorder that infers a `fill` from a +run of keystrokes has to guess where the run ends and which field had focus; a row that says +`screen.getByPlaceholder("Search transactions").fill("Paris")` guesses nothing. Secrets never reach +the file either, because the user types the value into the menu, not into the app. + +Still missing: + +- **Swipe**: a drag on the mirror records nothing. `pointerdown`→`move`→`up` over a threshold would + become `screen.swipe({ direction })`, or `locator.swipe(...)` when the start point resolves. +- **Typed-as-you-go**: key messages flow through `stream-input` and are not recorded at all, so a + flow where the user types into the app has to add the `fill` from the menu afterwards. + +### 6. The element menu: assertions and everything else + +A recorder that only taps produces worthless tests. The panel used to carry a global **Assert** +toggle, which armed the next tap to assert instead of acting. That is gone, and so are its tests: +one mode flag could express one extra step, and the moment typing, long-pressing and scrolling were +worth recording it would have needed one flag each. + +**A left click only drives the app.** It records nothing, recording or not. Tapping to find the +next screen used to add a step for every tap on the way, which the user then deleted, and it made +"drive the app" and "record a step" the same gesture. Steps come from the menu alone. + +**A row records the step and takes it.** A step recorded but never taken leaves the app on the +screen before it, so the next row would be resolved against a screen the test will not reach. `tap` +taps the element's own centre — where the runner's locator action lands, not the pixel that was +right-clicked — and `fill` taps the field and types. One gap: the runner's `fill` *replaces* a +field's contents and the mirror can only append, with no input message that clears a field, so a +`fill` into a field that already has text leaves the screen showing something the test will not +produce. The step is right; the screen behind it is not. `toBeVisible` and `scrollUntilVisible` do +nothing, because they change nothing: an assertion reads, and `scrollUntilVisible` is only offered +for an element the user just pointed at, which is on screen already. + +In its place is Maestro Studio's model. **Right-click an element on the mirror and the panel offers +what the runner can do with it**. The selected element is named once, followed by compact actions: + +``` +Selected element +"Go To Explore List" + +Tap +Is Visible +Scroll into view +────────────── +All commands › +``` + +**One row per kind of thing to do, not one per variant.** An earlier pass listed `longPress` beside +`tap` and `toHaveText` beside `toBeVisible`, which made the menu five near-duplicates of one +selector — harder to read than the three choices it was actually offering. The rest of what the +runner can express hangs off **All commands** as a submenu of its own, grouped into actions and +assertions, so `longPress`, `doubleTap`, `not.toBeVisible`, `toContainText`, `toHaveValue`, +`toBeEnabled`, `toBeDisabled` and `toBeSelected` no longer need the user to leave the panel and +write the line by hand. Only the short list is in front of them by default. + +**The submenu is a panel of its own and opens on hover.** Pointing at the row is enough — a click +and keyboard focus open it too — and it is drawn beside the menu rather than replacing what was in +it, so the selected element and the short list stay visible behind it. It always opens to the +right, the side its chevron points at. There is no flip to the left: this panel is often only a +few hundred points wide, so "flip when it does not fit" meant it opened on the left nearly always +— a chevron pointing one way and a panel opening the other. When the right has no room it is +pushed in from the window's edge instead, overlapping the menu it came from, as a cascading menu +does in a tight window; it is also narrower than that menu, so part of it stays visible behind. +The row and the panel are one hover target with a gap between them, so leaving either only starts +a close that entering the other cancels. + +A **step row's** own "Change command" is the same branch with `side: "left"`, which asks for the +left outright rather than only when the right has run out. It used to replace the step menu with +the command list and a Back row — navigation, not a menu: the step the user was acting on went +away, and the only way back was a button put there for the purpose. As a branch the step stays on +screen beside the commands. It opens on the left because the step list is the **rightmost** pane in +the workspace: to its right is the window's edge, and to its left is the preview, which is where +there is room. It still falls back to the right when the left cannot hold it, because a panel off +the edge of the window shows nothing at all. + +**Both popovers are portals fitted to the window, not to the stage.** The stage is a box with the +device in it and it clips what it contains, so a menu placed inside it was cut off at the bottom of +a short panel — and the old placement guessed the menu's height rather than measuring it. They are +rendered into `document.body`, positioned `fixed`, and measured through a `ResizeObserver` so a row +that opens a form re-fits the menu: below the element when the window has room, above it when it +does not, pinned to the edge they ran past when neither side fits, and scrolling only once they +are taller than the window itself. The arithmetic lives in `packages/ui/src/popover-placement.ts`, +apart from the component, so it can be checked against real layouts — a short panel, a narrow one +— rather than only through a DOM with no layout in it. They are free to lie over the mirror, the toolbar or the panel's edge — what +they must never be is clipped. Each is hidden until it has been measured, so neither is seen at the +position it was measured from, and because the anchor is read at measure time rather than passed as +numbers, a panel resized or scrolled under an open menu takes the menu with it. The canvas still +dismisses the menu when it is clicked through; a pointer down anywhere outside both popovers does +too, since neither is inside the canvas any more. + +The exact TypeScript remains on each row as a hover tooltip and appears in the live test preview +after selection. It is still produced by the same `stepExpression` used by the emitter, without +making the user scan the same selector once per action. Choosing an assertion row records the step +without touching the device; action rows such as tap and fill also perform the action so recording +can continue from the resulting screen. + +What is offered comes from the element's role and what it carries, never from the app: + +| Row | Offered when | +| --- | --- | +| `.tap()` | any element a selector names | +| `.fill("…")` | the role is `textbox`, `searchbox`, `combobox` or `spinbutton`, or the element has a placeholder — which only a text field does | +| `expect(…).toBeVisible()` | any element a selector names | +| `screen.scrollUntilVisible(…)` | any element a selector names | + +Under **All commands**, the same gating holds: `.fill("…")` and `toHaveValue` need a text field, +`toHaveText`/`toContainText` need an element with text of its own to start from, and everything +else is offered for any element a selector names. + +A row that needs a value asks for one in the menu rather than emitting an empty string literal: +`fill` starts from the field's current value, the text matchers from the element's own text. +Clicking an assertion row inserts it with e2e's default timeout. Hovering the row or clicking its +arrow opens **Insert** and **Add timeout**. Assertions that require expected text ask for it before +insertion. **Add timeout** opens a form with Back and Add controls, starting at e2e's 5,000 ms +default. Adding it writes the matcher's `{ timeout: ... }` option into the generated test; an +ordinary insert keeps the option implicit. + +Only actions the runner can really perform on a device are offered. `secondaryTap`, +`selectOption`, `setInputFiles`, `scrollIntoView` and tap `modifiers` all exist in the runner's +typings and all fail on a device with `UNSUPPORTED_CAPABILITY`, so they are not rows. +`check()`/`uncheck()` and `toBeChecked` are left out for the same reason by a different route: +the runner has them, and whether a device reports a `checked` state at all depends on the platform +and the component — an Android `Switch` does not — so a row for them would emit a step that cannot +be relied on to run, which is worse than a missing row. `doubleTap` is offered and is the one +command the panel cannot perform exactly as written: the input path has a single tap in it, so the +gesture is sent as two taps to keep the screen in step with the flow. + +One thing the menu needed fixing for: the recorder re-attaches its reader whenever the simulator +panel refreshes its device list, which is every few seconds, and `attach` used to invalidate the +cached tree every time. Through `clearHover` that took the outline — and any open menu — away about +a second after it appeared, which is exactly long enough to look like a flicker rather than a bug. +Only a reader that has actually changed invalidates now. + +The menu is anchored to the **element**, not to the pointer, so the wheel-driven parent walk still +applies: walk the outline out to the enclosing element and the menu re-asks what can be done with +the element it walked to, rectangle and rows together. + +It carries that rectangle itself rather than borrowing the hover outline's. The outline is dropped +by anything that invalidates the tree — an input, a frame, a read that is slow or fails, all of +which are ordinary while recording — and a menu the user asked for explicitly must not be taken +away with one. The menu's concurrency is its own too (`menuToken`): only another right-click or a +dismissal cancels it, never a hover that happens to land during its read. +A coordinate is offered the tap and nothing else. There is nothing about a point to be visible, to +hold text or to scroll to, which is the refusal the recorder already made for a point assertion. +The menu says so in a line above the row: three rows missing with no explanation reads as a broken +menu rather than as an answer. + +#### When nothing names the element uniquely + +The ranking prefers a unique candidate over an ambiguous one — depth first, rank within a depth, +and the first candidate that resolves to exactly one element wins whatever its rank, for the element +under the pointer or for one enclosing it. So reaching the ambiguous case means *every* candidate +was ambiguous, which happens for real: + +| element | role | unique selector | +| --- | --- | --- | +| the Settings **tab** | `tab` | `getByRole("tab", "Settings")` — matches 1 | +| the Settings **heading** | `text` | none | + +`getByRole` does not accept `text`, so the heading has only its own text to go on, and the tab +carries the same text. Nothing narrows it either: `filter({ hasText })` is a substring of the same +text, `filter({ has })` has no distinguishing child to point at, and there is no ancestor to scope +under — on this app every content node's parent is the application root. + +**So the step is narrowed by position.** `screen.getByText("Settings").nth(0)` runs on the device; +the bare selector fails with `LOCATOR_AMBIGUOUS: locator matched 2 nodes`. The index is the +position of the element the user actually pointed at among that selector's matches — the recorder +resolved that node, so it is recorded rather than guessed, and pointing at the *second* match +records `.nth(1)`. The match order is the snapshot's own: the projection flattens the tree depth +first and the panel passes it through untouched, so the panel's index and the runner's index are +the same index. + +The honesty stays where it belongs. A position is more brittle than a name, so: + +- the emitted line carries a note — not a TODO — saying how many elements match and that a testID + would pin it by name. Nothing has to be edited for the test to work; +- the step list reads `Expect visible "Settings" (1 of 2)` and still flags it `ambiguous`, and the + menu says how many match and that the step is pinned by position; +- the selector is still `ambiguous` as far as the panel is concerned, because it is. + +The alternative was a step that fails and a note asking the user to go and edit it. A recorded test +that runs, with the user told exactly what is weak about it, is the better of the two. + +### 7. Emitting and writing + +`RecordedStep[]` is the single source of truth. Emitters are pure functions over it: + +```ts +interface StepEmitter { + readonly id: "e2e" | "maestro"; + emit(steps: RecordedStep[], meta: { title: string }): string; +} +``` + +The e2e emitter produces exactly the shape of a hand-written deterministic test: + +```ts +import { test } from "@e2e-dev/mobile"; +import { expect } from "e2e"; + +test("", async ({ app, screen }) => { + await app.open(); + await expect(screen.getByRole("tab", "Home")).toBeSelected(); + await screen.getByRole("tab", "Explore").tap(); + // … +}); +``` + +No `agent.act` is ever emitted — the whole point of recording is determinism. + +Writing needs a new **app-only** MCP tool in the server process: + +- `mobile_testing_write_file({ projectRoot, relativePath, contents })` +- Annotated as a write so the host prompts; the panel must not be able to write anywhere by + accident. Constrain `relativePath` to `tests/` under the chosen root, reject `..`, refuse symlinks. +- **It fires only when the user clicks Save, or when Run saves for them.** One tool call, one host + prompt, per save — not per tap. This is what makes the permission story tolerable. + +Running needs a second one, `mobile_testing_run_test({ projectPath, relativePath, timeoutMs? })`, +annotated destructive because a run drives the app and changes its state — so a host that prompts +on destructive tools prompts once per Run, which is the right granularity: a run is an explicit act, +not something that happens as you tap. It releases the recorder's +device session, spawns the **project's own** `node_modules/.bin/e2e` (never `npx`, which would go +to the network for a package that is not installed), and returns `{ status, exitCode, steps[] }` +with each step's `line`, `status` and first line of error. + +While recording, the panel holds `RecordedStep[]` and renders the emitted source as a **live +preview** beside the step list, syntax-coloured by a tokenizer written for exactly the file the +emitter produces (`packages/ui/src/code-highlight.ts`) rather than a highlighting library — the +panel ships as one HTML file and a grammar would dwarf what it renders. + +The lines a step adds are **tinted as they arrive** and fade back into the file. The user is +looking at the mirror when a step is recorded, not at the preview, so the new lines have to be +findable a second later. The file is re-emitted whole on every change, so "what is new" is +recovered by matching the previous render from both ends (`changedLines`); the marked lines are +keyed by a generation so each step animates, not only the first. The preview is always a full re-emit from the steps, so editing or +deleting a step just re-renders; nothing is ever half-written. + +**Test Description** is the collapsible section the title field, the file name field and Save sit +in. They are touched once per recording — at the end — while the step list beside them is touched +on every tap, so permanently open they cost a third of a pane that is mostly used for the steps. +Closed, the row is the readout: the title and the path the flow will be saved under, which is what +someone who has not opened it needs to know. The name has a **text width of its own** and clips +inside it, so the row degrades the same way wherever the pane ends, and carries the whole value as +a tooltip. The file name is **not** beside it: at the width this pane actually gets, a name and a +path together were two truncated strings reading as neither — `a recorded flow lkfl… · +tests/a-recorded-flow-lkflksd-s…`. The path is one click away in the File name field, which is +where it is edited anyway. + +**Clear asks first, and is the icon alone.** It throws away every step of the flow on screen and +there is no undo — the steps are the whole of what was recorded — so a button labelled "Clear" at +the end of the step count's row was a destructive default one click deep. It is now the trash icon +tinted as the destructive thing it is, and the question it asks names how many steps are about to +go. + +Save behaviour: + +- **Record starts a new flow** when the steps on screen are already saved, and continues the + existing one when they are not — stopping to scroll and carrying on is still one flow, but a tap + landing on the end of a finished test never is. +- **A successful save stops the recording.** The flow is written; leaving the recorder armed turns + the next stray tap on the mirror into a step the saved file does not have. The steps stay, so the + flow can be extended and saved again. A save blocked by an existing file changes nothing. +- A filename field, defaulted from the test title (`tab-navigation` → `tests/tab-navigation.e2e.ts`). +- If the target exists, the panel says so and the user confirms an overwrite before the tool runs. +- The panel tracks a dirty flag: steps recorded since the last Save. Stopping or switching projects + with unsaved steps warns. + +Alternative considered and rejected: debounced live-write after every tap. It is what Maestro Studio +feels like, but each write is an MCP tool call and so a host permission prompt — unusable in a +recorder where a flow is thirty taps. + +### 8. Replay + +**Run test** saves the flow, hands the device back, runs the file with the project's own `e2e` CLI, +and marks each line of the preview with what the runner reports for it. + +```mermaid +sequenceDiagram + participant P as Panel + participant S as Server + participant R as e2e CLI + P->>S: mobile_testing_write_file (asks before replacing) + P->>S: mobile_testing_release_device + P->>S: mobile_testing_run_test + S->>S: devices.release() again + S->>R: node_modules/.bin/e2e run <file> --reporter json + R-->>S: the whole report, at the end + S-->>P: steps[] with source.line and status + P->>P: marks per line, held then faded on a pass +``` + +**Run saves.** The runner runs a file, so there has to be one, and running a copy in a temporary +path would run a test whose failures point at lines of a file the user cannot open. It saves +through the same path the Save button uses, so an existing file is still never replaced without +being asked about: the overwrite question stops the run, and answering it resumes the run rather +than making the user press Run twice. + +**Run releases the device first, and waits.** agent-device allows exactly one session per device +and the recorder holds it, so a run spawned while it is held fails with `Device is already in use +by session "mobile-dev-recorder"` — a refusal the user cannot connect to anything they did. The +panel stops recording, awaits `mobile_testing_release_device`, and only then runs; the server tool +releases again on its own account, because a caller who forgets produces exactly that failure. +The release is a dependency of the run (`TestRunDeps` in `packages/core/src/testing-run.ts`), not a +step a caller is trusted to have taken, and it happens only after the runner binary resolves — a +project with no runner installed must not cost the user their recording session. + +#### Where live per-step progress comes from + +None of the CLI's four reporters gives it: + +| Source | What it gives | +| --- | --- | +| `--reporter list`, TTY | deterministic steps appear only inside a live window it repaints | +| `--reporter list`, no TTY | a permanent line per **agent** step; a `locator.tap` or an `expect` never prints | +| `--reporter json` | the whole report document once, at the end | +| `.e2e/report.json` | written from `onRunFinished`; it does not exist mid-run | + +But the runner's **event spine** does, and it is reachable without changing the runner: +`e2e.config.ts` accepts reporter *objects*, and a reporter's `onEvent` receives every `step` event +live, in the runner process, with `phase: "start" | "end"`. `--reporter` replaces only the built-in +reporters, so a configured object still runs beside `json`. + +So a run is given a **generated config** that spreads the project's own and appends one reporter: + +```mermaid +flowchart LR + Panel -->|mobile_testing_run_test| Server + Server -->|writes| Cfg[".e2e/mobile-dev-progress.config.mts"] + Server -->|e2e run --config| Runner + Cfg -.->|spreads| Base["the project's e2e.config.ts"] + Runner -->|onEvent, tagged line on stderr| Server + Panel -->|mobile_testing_run_progress, polled| Server +``` + +- The config is written **beside the project's own**, at the project root, and removed when the run + ends — including on a timeout or a spawn failure. Not in `.e2e/`, tidy as that would be: the + runner takes the project root from the config file's own directory, so a config one level down + ends the run in `NO_TESTS: no test file matched "tests/**" under <project>/.e2e`. +- It spreads the project's config, resolving a config that exports a function, and puts its own + reporters first. It adds a reporter and changes nothing else. +- Reporter **construction** has no side effects, because workers load the config too. Every write + happens inside `onEvent`, wrapped in `try`, since a reporter that throws is quarantined for the + rest of the run. +- Events come back as `@@mobile-dev-step <json>` lines on the child's stderr. No socket, no second + temp file, and the runner's own stderr passes through untouched. +- A tool call is one request and one response, so the panel polls `mobile_testing_run_progress` + every 400 ms while the run is in flight. +- If the runner refuses the generated config — an extension its loader will not take, a project + config that does not survive being spread — the run is retried exactly as it was before live + progress existed. A spinner is worth having and is not worth losing a run over. + +**Mapping an event to a line.** A `step` event carries no source position — only the finished +report does — but it carries `identity.stepIndex`, and the runner reports one step per awaited call +in order. `EmittedTest.calls` is every awaited line in file order, `await app.open()` first, so the +index *is* the position in that list. The report reconciles it at the end, where each step carries +its own `source.line`, so a mismatch corrects itself rather than persisting. + +The states are then: + +- **nothing marked** until the runner says where it is. A spinner on every line would claim more + than is known. +- **one line spinning** — the step that has started and not ended. +- **a mark behind it** for each step that has ended, with its own status. +- a pass holds its checkmarks for 2.5s and fades them; a failure's mark stays until the next run. + Both respect `prefers-reduced-motion`, which drops the transition and keeps the mark. + +The gutter is a **grid column**, not a character at the front of the line: each preview line is +`grid-template-columns: 1.35em minmax(0, 1fr)` with the mark and the code as siblings, and the mark +is exactly `1lh` tall so it centres on its line. That is what keeps the code from shifting when a +run starts and keeps a mark level with the first row of a line that wraps — and a long locator does +wrap. + +If a run produces no report at all — the runner refused before it reached the test — there is no +line to mark, so the panel shows the runner's own output instead of going quiet. + +#### Line spans + +Attributing a report to a line needs to know which lines a step produced, and a step is not one +line: an ambiguous step carries a TODO above its call, and a coordinate tap carries a TODO *and* an +anchor gate, which the runner reports as a step of its own. `emitE2eTestLines` returns +`{ source, spans }`, where each span is `{ stepId, from, to, calls }` in 1-based file lines and +`calls` are the lines the runner will report a step for. `emitE2eTest` is that function's `source`, +so the preview, the saved file and the spans can never describe different files. The preview's +new-line animation still uses `changedLines` — good enough to flash a line, far too weak to decide +a pass or a failure. + +## Reading the runner's tree + +**The recorder reads the element tree the runner will query, not a second tree over the same +pixels.** `packages/core/src/testing-device.ts` takes an agent-device snapshot of the device — +`capture.snapshot({ interactiveOnly: false })`, the same call the runner's own locator path makes +— and projects it with the **project's own** `@e2e-dev/mobile` build, the one that will run the +test. The panel reaches it through `mobile_testing_read_tree`. + +This replaces reading the macOS accessibility API through Baguette and translating its vocabulary +into the runner's with a table in `test-recording.ts`. That table was the bug, not the fix for it. +Two trees over the same pixels do not agree, and every way they disagreed reached the user as a +recorded test that simply did not resolve: + +| | What iOS accessibility said | What the runner says | +|---|---|---| +| Tab bar item | `AXRadioButton` — a table had to guess `tab` | `tab`, derived from the tab bar around it | +| `<Text>` inside a `<Link>` | `AXLink`, from the link *trait* that any view can carry | `text`, from the XCUIElement *type* | +| Search field | name "Search transactions" (its placeholder) | no name; `placeholder` is "Search transactions" | +| A line of text and its wrapper | two matches, so "ambiguous" | one match, innermost | + +Reading one tree makes all four agree **by construction** rather than by a mapping someone has to +maintain: role vocabulary and name sources are the engine's, and the recorder only chooses which +query to use. `normalizeRole`, `ROLE_ALIASES` and the fiber-versus-accessibility split are gone +with it, and so is the simulator-chrome filter — the runner's snapshot is rooted at the +application, so the clock and the Dynamic Island are not in it to filter. + +It costs a snapshot per read (about 0.1 s in practice, measured against a booted iPhone 17 Pro), +against tens of milliseconds for a Baguette loopback. A tap holds the touch until the read +returns, and the recorder caches the tree for 1.5 s and invalidates it on every input, so a warm +tree answers a hover or a walk without a round trip. + +### Baguette still owns the mirror + +Only the recorder's *read* path moved. Video frames and touch injection are unchanged, and the +two hold the same simulator at once: with the mirror streaming, a snapshot returns in about a +tenth of a second, the stream stays open and a frame arrives for the change the snapshot's own tap +caused. + +### The session, and giving it back + +Reading holds an agent-device session on the device, and **agent-device allows exactly one session +per device**. A session left open makes the user's next `npx e2e run` fail with `Device is already +in use by session …` — a refusal they cannot connect to anything they did, and a worse failure +than any this path fixes. So the device is released on every path out of recording: stopping, +detaching, a device that changes under the recorder, the panel being disposed, and **Run**, which +waits for the release before it spawns anything (`RecorderReader.release`, +`TestingDevices.release`). + +A session is also released when the **server shuts down**, which is the path none of those cover. +The claim record agent-device writes names the daemon that holds it, and that daemon outlives the +server process. A server that dies without releasing therefore leaves a claim behind on a daemon +nobody is talking to any more, and the user's own `e2e run` then refuses to boot the device: + +``` +ERROR infrastructure error ENGINE_FAILURE (launch) +boot failed: iPhone 17 Pro has an ownership claim that could not be verified. +``` + +This is the same leak as `Device is already in use`, wearing a worse message, and the user has even +less to connect it to — the panel they closed is gone. `registerTestingTools` returns the release as +the plugin's shutdown for that feature, and `plugin.close()` awaits it beside the other collectors. +When it does happen (a hard kill releases nothing), the way out is the CLI the daemon was started +by, on the same state directory: + +```bash +AGENT_DEVICE_STATE_DIR="$HOME/Library/Application Support/mobile-dev/app-flow/agent-device" \ + <plugin>/dist/agent-device/node_modules/.bin/agent-device close --session mobile-dev-recorder +``` + +Attaching names the bundle id and passes `relaunch: false`. Never `--foreground`, which refuses +outright when more than one app is running on the simulator, and never anything that relaunches: +the recorder exists so the user keeps tapping the screen they are already on. + +## Maestro, later + +Nothing above is e2e-specific except the emitter and the run command. Adding Maestro means: + +- a `MaestroEmitter` producing YAML (`- tapOn: { id: … }`), +- a run path through the existing agent-device script-replay tool, +- a format switch in the panel. + +The selector ranks map cleanly: `identifier` → `id`, text → `text`, point → `point`. + +## Resolved against e2e v0.17.0 + +- **`screen.tapAt(point, options?)` exists** (`e2e/dist/types.d.ts:296`), with the point in CSS + pixels. Rank 4 of the selector table is valid as written. +- **No `e2e config --json`.** The CLI has `init`, `login`, `logout`, `models`, `guide`, `mcp`, + `run`, `explore`, `list`, `cache`, `feedback`, `telemetry`. `e2e list --reporter json` prints + `{ pairs: [...] }` — test/target pairs, which confirms a folder is an e2e project and names its + targets, but does not expose the bundle id. + So: validate the project with `e2e list --reporter json`, and get the bundle id by reading + `e2e.config.ts` with a light regex, falling back to asking the user once and caching it. + Parsing the config properly would mean executing TypeScript in the server process — not worth it. + +## Open questions + +None blocking. + +## Build order + +1. ~~Tab + empty panel registered in all hosts.~~ Done: `testing` in `toolNames` + (`packages/ui/src/workspace-tools.ts`), the tab in `packages/mcp-app/src/components/workspace.tsx`, + `TestingPanel` (`packages/ui/src/testing-panel.ts`) and `TestingView` + (`packages/ui/src/components/testing-view.tsx`). +2. ~~Project picker + launch, no recording.~~ Done: `mobile_testing_projects`, + `mobile_testing_choose_project` and `mobile_testing_project_config` + (`packages/core/src/testing-tools.ts`, `packages/core/src/testing-projects.ts`), with launch + going through the existing `mobile_launch_app`. +3. ~~Tap recording with selector resolution, step list and live source preview in the panel.~~ + Done: `TestRecorder` (`packages/ui/src/test-recorder.ts`) with the pure selector and emitter + logic in `packages/shared/src/test-recording.ts`, taps intercepted on the mirror canvas in + `packages/ui/src/simulator-panel.ts`, and the step list and preview in `TestingView`. + Right-click on the mirror opens the element menu (step 5), which records the steps a tap + cannot express without forwarding a touch. +4. ~~`mobile_testing_write_file` + the Save button and overwrite confirmation.~~ Done: + `writeTestFile` and `resolveTestPath` (`packages/core/src/testing-projects.ts`) confine the + write to the project's `tests/`, refuse `..`, a non-`.e2e.ts` name and a symlinked folder or + file, and report an existing file back instead of replacing it; `TestingPanel.saveTest` + asks before overwriting and tracks saved-versus-dirty against the recorder's revision. +5. ~~Assertions, typing, long press, scroll — through the element menu.~~ Done: `stepOffers` and + `stepExpression` (`packages/shared/src/test-recording.ts`) derive the rows from the element's + role and state; `TestRecorder.openMenu`/`chooseOffer` resolve and record them at the outline's + walked-out level; `ElementMenu` (`packages/ui/src/components/test-recorder-overlay.tsx`) draws + them over the mirror from the `contextmenu` wired in `packages/ui/src/simulator-panel.ts`. The + global Assert toggle and its tests are gone. +6. ~~Run button.~~ Done: `mobile_testing_run_test` and `runRecordedTest` + (`packages/core/src/testing-tools.ts`, `packages/core/src/testing-run.ts`) release the device as + a dependency of the run and turn the report into per-line status; + `TestingPanel.runTest` saves first and resumes from the overwrite answer; the marks are + `TestRunView` on the recorder and the gutter in `TestPreview`. +7. Swipe recording (a drag on the mirror, which currently records nothing). +8. Maestro emitter. diff --git a/packages/core/src/app-flow-host.ts b/packages/core/src/app-flow-host.ts index 0a2d5207..baa767ce 100644 --- a/packages/core/src/app-flow-host.ts +++ b/packages/core/src/app-flow-host.ts @@ -544,16 +544,18 @@ interface MutableRun extends AppFlowRunSnapshot { event?: AppFlowHostEvent; } -function vendoredAgentDeviceRoot(): string | undefined { +export function vendoredAgentDeviceRoot(): string | undefined { const roots = [ join(dirname(fileURLToPath(import.meta.url)), "agent-device/node_modules/agent-device"), resolve(process.cwd(), "runtimes/agent-device/node_modules/agent-device"), resolve(process.cwd(), "../ai-harness/node_modules/agent-device"), + // Running from source rather than from a packaged dist/, where the vendored runtime is a workspace. + resolve(process.cwd(), "packages/runtimes/agent-device/node_modules/agent-device"), ]; return roots.find(root => existsSync(join(root, "dist/src/index.js"))); } -async function loadVendoredAgentDevice(): Promise<{ createAgentDeviceClient(config?: Record<string, unknown>): any }> { +export async function loadVendoredAgentDevice(): Promise<{ createAgentDeviceClient(config?: Record<string, unknown>): any }> { const root = vendoredAgentDeviceRoot(); const entry = root ? join(root, "dist/src/index.js") : undefined; if (!entry) throw new Error("The bundled agent-device runtime is unavailable. Reinstall or rebuild Mobile Dev."); diff --git a/packages/core/src/baguette.ts b/packages/core/src/baguette.ts index f3a05602..0329b6a9 100644 --- a/packages/core/src/baguette.ts +++ b/packages/core/src/baguette.ts @@ -22,6 +22,13 @@ import type { ResourceSaving } from "./resource-saving.ts"; type DescribedUi = { tree?: { label?: unknown } }; const NO_ACCESSIBILITY_DATA = "no accessibility data"; +/** + * The root of a healthy tree is the foreground app, and it is named. A bridge that has gone stale + * answers with an unnamed root carrying only the simulator's chrome, so the name is the signal. + */ +function unnamedRoot(described: DescribedUi): boolean { + return typeof described.tree?.label !== "string" || described.tree.label.trim() === ""; +} const BRIDGE_RESTART_INTERVAL = 10000; const PORT_ATTEMPTS = 3; // A failed device list keeps its request's error, so errors built from the status say how the request failed. @@ -119,16 +126,24 @@ export class Baguette { async describeUi(udid: string): Promise<DescribedUi> { const path = `/simulators/${udidSchema.parse(udid)}/describe-ui.json`; - try { return await this.json(path) as DescribedUi; } + let described: DescribedUi | undefined; + try { described = await this.json(path) as DescribedUi; } catch (error) { if (!(error instanceof Error) || error.message !== NO_ACCESSIBILITY_DATA || !await this.restartAccessibilityBridge(udid)) throw error; } + // A stale bridge does not always fail outright. It answers `ok`, with the simulator's own + // status bar and an unnamed root where the app's hierarchy should be — the same condition the + // restart below exists for, reported as success. Callers downstream cannot tell that apart + // from an app that genuinely has no accessible content, so it is caught here. + if (described && !unnamedRoot(described)) return described; + if (described && !await this.restartAccessibilityBridge(udid)) return described; try { const result = await this.json(path) as DescribedUi; recordAccessibilityBridgeRestart("recovered"); return result; } catch (error) { recordAccessibilityBridgeRestart("unavailable"); + if (described) return described; throw error; } } diff --git a/packages/core/src/plugin.ts b/packages/core/src/plugin.ts index f8f678ec..905196ac 100644 --- a/packages/core/src/plugin.ts +++ b/packages/core/src/plugin.ts @@ -18,6 +18,12 @@ import { registerAppStorageTools } from "./app-storage/tools.ts"; import { APP_FLOW_DIRECTORY, createAppFlowHost } from "./app-flow-host.ts"; import { AppFlowProjectChoices } from "./app-flow-projects.ts"; import { registerAppFlowTools } from "./app-flow-tools.ts"; +import { registerTestingTools } from "./testing-tools.ts"; +import { registerTestingAgentTools } from "./testing-agent.ts"; +import { TestingDevices } from "./testing-device.ts"; +import { TESTING_DIRECTORY, TestingProjectChoices } from "./testing-projects.ts"; +import { TestDraftStore } from "./testing-drafts.ts"; +import { TestingAgentRecordings } from "./testing-agent.ts"; import { ServeEmu } from "./serve-emu.ts"; import { listIosDevices, registerIosDeviceTools } from "./ios-devices.ts"; import { deviceChoiceFormSupport, registerDeviceChoiceTools } from "./device-choice-tools.ts"; @@ -113,6 +119,14 @@ export async function createPlugin(html: string | (() => Promise<UIResource>), o registerAppIntelligenceTools(server, appIntelligence); const appFlow = await createAppFlowHost(); const closeAppFlow = registerAppFlowTools(server, appFlow, new AppFlowProjectChoices(APP_FLOW_DIRECTORY)); + // One device holder for both drivers. agent-device allows exactly one session per device, and an + // agent recording is another holder of the panel's `mobile-dev-recorder` session, never a second. + const testingDevices = new TestingDevices(); + // One store for both: the panel autosaves the flow it is holding and the agent keeps the flow it + // finished, and those are the same list of unsaved recordings. + const testDrafts = new TestDraftStore(); + const closeTestingDevices = registerTestingTools(server, new TestingProjectChoices(TESTING_DIRECTORY), testingDevices, testDrafts); + registerTestingAgentTools(server, testingDevices, new TestingAgentRecordings(testingDevices, testDrafts)); const resource = async () => typeof html === "string" ? { html } : await html(); const page = async () => configureUI((await resource()).html); @@ -161,7 +175,7 @@ export async function createPlugin(html: string | (() => Promise<UIResource>), o stopRecordingStorageMetrics(); // Recordings finish while the collectors they read are still open. await recordings.close(); - try { await Promise.all([logs.close(), cpu.close(), fps.close(), traceSift.close(), closeAppFlow(), closePanel?.()]); } + try { await Promise.all([logs.close(), cpu.close(), fps.close(), traceSift.close(), closeAppFlow(), closePanel?.(), closeTestingDevices()]); } finally { android.dispose(); await server.close(); await analytics.close(); } }, }; diff --git a/packages/core/src/react-native-snapshot.js b/packages/core/src/react-native-snapshot.js index 08fc80a3..3e0073a1 100644 --- a/packages/core/src/react-native-snapshot.js +++ b/packages/core/src/react-native-snapshot.js @@ -16,6 +16,33 @@ export function collectReactNativeTree(complete) { } const clean = value => typeof value === "string" ? value.slice(0, 256) : undefined; const componentName = type => clean(typeof type === "string" ? type : type?.displayName ?? type?.name ?? type?.render?.displayName ?? type?.render?.name ?? type?.type?.displayName ?? type?.type?.name); + // A <Text> holding nested <Text> has no string child of its own: `children` is an array of + // strings and elements, so reading it directly yields nothing and the node falls back to its + // component name. iOS accessibility does not cover React Native views, so this string is the + // only name such an element will ever have — flatten it. Only text components are entered, so a + // View never collects its whole subtree's copy. + const TEXTUAL = /Text$/; + function flattenText(value, depth) { + if (typeof value === "string") return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + if (depth >= 6 || !value) return ""; + if (Array.isArray(value)) { + let joined = ""; + for (const item of value) { + joined += flattenText(item, depth + 1); + if (joined.length > 512) break; + } + return joined; + } + const name = value.type === undefined ? undefined : componentName(value.type); + return name && TEXTUAL.test(name) ? flattenText(value.props?.children, depth + 1) : ""; + } + function childText(children) { + try { + const flat = flattenText(children, 0).replace(/\s+/g, " ").trim(); + return flat ? clean(flat) : undefined; + } catch { return undefined; } + } const sourceUrls = [], sourceStacks = [], stackIds = new Map(); function creationStack(fiber) { const error = fiber?._debugStack ?? fiber?.debugStack; @@ -73,8 +100,12 @@ export function collectReactNativeTree(complete) { const name = componentName(type); let node = parent; if (name && fiber.tag !== 10 && fiber.tag !== 9) { - const childText = typeof props?.children === "number" && Number.isFinite(props.children) ? String(props.children) : clean(props?.children); - node = { source: "react-native", role: name, label: clean(props?.accessibilityLabel) ?? childText ?? name, identifier: clean(props?.testID), nodeId: `rn-${nodes.length}`, parentId: parent?.nodeId, depth, children: [], ...reactInfo(fiber, name) }; + // Only a text component flattens its nested text. A Pressable holding a <Text> keeps no + // label of its own: its accessible name is accessibility's business, and inventing one here + // would name a container after the copy inside it. + const text = TEXTUAL.test(name) ? childText(props?.children) : clean(props?.children) + ?? (typeof props?.children === "number" && Number.isFinite(props.children) ? String(props.children) : undefined); + node = { source: "react-native", role: name, label: clean(props?.accessibilityLabel) ?? text ?? name, identifier: clean(props?.testID), nodeId: `rn-${nodes.length}`, parentId: parent?.nodeId, depth, children: [], ...reactInfo(fiber, name) }; parent?.children.push(node); nodes.push(node); if (instance) readNativeChildren(instance.ownerDocument?.documentElement ?? instance); diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index a2ce4e3d..e4ab0d36 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -79,6 +79,21 @@ export function recordProjectDialog(outcome: "chosen" | "cancelled" | "failed", Sentry.metrics.distribution("app_flow.project_dialog.duration", Math.max(0, duration), { unit: "millisecond", attributes }); } +/** + * How long an agent recording's screen read took, and how much of a screen it answered with. + * + * The read is the only thing in an agent recording that costs anything — a flow is ten to twenty + * of them — so it is the boundary worth measuring. The element count is a size, never content: + * no names, no text and nothing the user's app put on screen is sent. + */ +export function recordTestingAgentRead(outcome: "success" | "failed", duration: number, elements: number, platform: "ios" | "android") { + if (process.env.MOBILE_DEV_TELEMETRY === "off") return; + const attributes = { component: "server", surface: "testing", platform, outcome }; + Sentry.metrics.count("testing.agent.screen.samples", 1, { attributes }); + Sentry.metrics.distribution("testing.agent.screen.duration", Math.max(0, duration), { unit: "millisecond", attributes }); + if (outcome === "success") Sentry.metrics.gauge("testing.agent.screen.elements", elements, { attributes }); +} + export function recordTraceSiftOperation(operation: "ready" | "analyze" | "remember" | "restore" | "handoff" | "task" | "payload" | "pick" | "project", outcome: "success" | "failed", duration: number) { if (process.env.MOBILE_DEV_TELEMETRY === "off") return; const attributes = { component: "server", surface: "tracesift", operation, outcome }; diff --git a/packages/core/src/testing-agent.ts b/packages/core/src/testing-agent.ts new file mode 100644 index 00000000..e8aee672 --- /dev/null +++ b/packages/core/src/testing-agent.ts @@ -0,0 +1,637 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; +import { z } from "zod"; +import { errorMessage } from "@mobile-dev/shared/protocol.ts"; +import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; +import { deviceIdSchema } from "@mobile-dev/shared/panel-target.ts"; +import { + agentStepLabel, anchorSelector, emitE2eTest, resolveComponentSelector, selectorSource, stepEffect, stepExpression, + stepLabel, stepNeedsValue, swipeDirection, swipePath, +} from "@mobile-dev/shared/test-recording.ts"; + +import type { + AgentStep, AgentStepKind, PinnedStep, PinnedStepKind, RecordedStep, StepTarget, SwipeDirection, +} from "@mobile-dev/shared/test-recording.ts"; +import type { ScreenComponent, ScreenSize } from "@mobile-dev/shared/screen-annotations.ts"; +import { TestingDevices } from "./testing-device.ts"; +import { TestDraftStore } from "./testing-drafts.ts"; +import type { TestingDeviceTarget } from "./testing-device.ts"; +import { captureServerError, recordTestingAgentRead } from "./telemetry.ts"; + +/** + * Recording an e2e test from the user's own coding agent, into the Testing panel the user is + * looking at. + * + * The manual recorder lives entirely in the iframe — `testRecorder` is a singleton there and + * `RecordedStep[]` exists nowhere else — and an agent talks to this process over MCP, not to that + * object. The server is the only thing the two share, so an agent's recording lives here and the + * panel reads it, which is the same shape as run progress and for the same reason: a tool call is + * one request and one response, so anything live has to be polled. + * + * The reasoning stays with the agent. There is no model in this process, one step per tool call, + * and nothing here decides what is worth recording — see `docs/agent-recording.md`. + */ + +/** The ranks a screen read reports, in the order `resolveChain` tries them. */ +const RANK_LABEL: Record<string, string> = { + testId: "testId", role: "role + name", placeholder: "placeholder", text: "text", label: "name", + nth: "position among matches", point: "coordinate", +}; + +/** + * How many elements one screen read answers with. + * + * A screen of review rows runs to several hundred nodes and the agent only ever names one of them, + * so the read is cut rather than left to fill a context window. It is a cut, not a filter: the + * rows are in the snapshot's own depth-first order, so what is dropped is the end of the tree and + * the read says how many it dropped. + */ +const MAX_ELEMENTS = 200; + +/** What the panel's selection is, as the panel reports it. Nothing here is inferred. */ +export type TestingAgentContext = { projectPath: string; deviceId: string; platform: "ios" | "android"; bundleId: string }; + +export type TestingAgentStatus = "idle" | "recording" | "finished" | "aborted"; + +/** What the panel polls for, and the whole of what it draws the agent banner from. */ +export type TestingAgentState = { + active: boolean; + status: TestingAgentStatus; + /** Bumped by every change; the panel adopts the steps when it moves. */ + revision: number; + title: string; + goal: string; + steps: RecordedStep[]; + note: string; + error: string; + /** + * The unsaved recording the finished session was kept as, or "" while it is still in flight. + * + * The server writes the draft rather than the panel, because the panel may never have been + * looking: the user asks their agent for a test in the chat, reads the answer there, and only + * opens the Testing tab later. The panel adopts this id so its own autosave keeps replacing the + * same draft instead of leaving a second copy of the same flow in the list. + */ + draftId: string; +}; + +type Session = { + target: TestingDeviceTarget; + title: string; + goal: string; + steps: RecordedStep[]; + status: Exclude<TestingAgentStatus, "idle">; + note: string; + error: string; + /** + * Which read the live refs came from. Every action invalidates the tree, so a `ref` carries the + * read it was minted by and one from an older read is refused rather than acted on — the + * manual recorder makes the same assumption, it just has a pointer to re-hover with. + */ + readId: number; + refs: Map<string, { ref?: string; component: ScreenComponent }>; + components: ScreenComponent[]; + screen?: ScreenSize; + /** The rows of the last read, kept so a read the step path already paid for is not paid twice. */ + rows: ScreenRow[]; + /** How many elements that read left out, for the same reason. */ + truncated: number; + /** When the last read happened, so a tree the app has since changed on its own is not served. */ + readAt: number; + /** The last read's element identities; see `hasNavigated`. */ + identities: Set<string>; + /** Set once the session ends and its steps are kept as an unsaved recording. */ + draftId: string; +}; + +/** One element of a screen read, as the agent sees it. */ +type ScreenRow = { + ref: string; + role?: string; name?: string; text?: string; placeholder?: string; value?: string; testId?: string; + enabled?: boolean; selected?: boolean; offScreen?: boolean; + bounds: number[]; + selector?: string; rank?: string; unique?: boolean; +}; + +const PINNED_KINDS = [ + "tap", "doubleTap", "longPress", "fill", "scrollUntilVisible", "assertVisible", "assertNotVisible", + "assertText", "assertContainsText", "assertValue", "assertEnabled", "assertDisabled", + "assertSelected", "swipe", +] as const satisfies readonly PinnedStepKind[]; +const AGENT_KINDS = ["agentAct", "agentAssert", "agentWaitFor"] as const satisfies readonly AgentStepKind[]; + +/** + * How long a read stays usable without touching the device. + * + * An acting step reads the tree straight after its action — it has to, to tell the agent whether + * the screen was replaced — and the agent's own next `agent_screen` would otherwise read the same + * quiet screen again. This bounds the one thing that can still have changed: the app moving on by + * itself, an animation finishing or data arriving. The manual recorder caches on the same rule. + */ +const TREE_TTL_MS = 1_500; + +/** + * What an element is, independently of the read it came from. + * + * Node ids are minted per read, so two reads of one unchanged screen share none of them. What a + * test would name an element by — its role, its testID, its name, its text — is stable across + * reads and is exactly what the agent is looking at. + */ +function identity(component: ScreenComponent): string { + return [component.role, component.identifier, component.label, component.text, component.placeholder] + .map(value => (value ?? "").replace(/\s+/g, " ").trim()).join("\u0000"); +} + +/** + * Whether a step left the screen it was recorded on. + * + * Measured as the overlap between the two reads' element identities, because that is the thing + * the rule is actually about: a step that *changes* a screen leaves most of it in place — a + * disclosure adds a few rows, a filter rewrites a list under a heading, a fill re-labels one field + * — while a step that *navigates* replaces it. Measured against a real app's Explore screens: + * opening a disclosure kept every element the read had, switching tabs kept about two fifths of + * them (the tab bar and the chrome around it), and tapping into a pushed screen kept about a + * tenth. A third sits in the gap, with room on both sides. + * + * A tab switch therefore does not fire, which is the answer worth having: it is usually the + * navigation *to* the screen the recording is for, so a boundary warning there would land exactly + * as the agent arrives. And it is a hint either way — nothing refuses a step because of it. + */ +function hasNavigated(before: Set<string>, after: Set<string>): boolean { + if (before.size === 0 || after.size === 0) return false; + let kept = 0; + for (const value of before) if (after.has(value)) kept += 1; + return kept / before.size < 1 / 3; +} + +function refused(message: string): never { + throw new ExpectedOperationError("invalid_input", message); +} + +/** Only the fields a command row is gated on; see `PinnedStep.target`. */ +function stepTarget(component: ScreenComponent): StepTarget { + return { + ...(component.role === undefined ? {} : { role: component.role }), + ...(component.placeholder === undefined ? {} : { placeholder: component.placeholder }), + ...(component.label === undefined ? {} : { label: component.label }), + ...(component.text === undefined ? {} : { text: component.text }), + ...(component.value === undefined ? {} : { value: component.value }), + }; +} + +/** + * The one agent recording, and the panel selection it records against. + * + * One at a time, for the same reason the manual recorder is a singleton: a test is one flow on one + * device, and two sessions would interleave two screens into one file. A second `begin` is refused + * rather than allowed to stomp on the first. + */ +export class TestingAgentRecordings { + private readonly devices: TestingDevices; + private readonly drafts: TestDraftStore; + private context?: TestingAgentContext; + private session?: Session; + private revision = 0; + + constructor(devices: TestingDevices, drafts = new TestDraftStore()) { this.devices = devices; this.drafts = drafts; } + + /** The panel's current project, device and bundle id. The only thing that knows the selection. */ + setContext(context: TestingAgentContext) { this.context = context; } + clearContext() { this.context = undefined; } + + state(): TestingAgentState { + const session = this.session; + return { + active: session?.status === "recording", + status: session?.status ?? "idle", + revision: this.revision, + title: session?.title ?? "", + goal: session?.goal ?? "", + steps: session?.steps ?? [], + note: session?.note ?? "", + error: session?.error ?? "", + draftId: session?.draftId ?? "", + }; + } + + private changed() { this.revision += 1; } + + private recording(): Session { + const session = this.session; + if (session?.status !== "recording") { + refused("No agent recording is in flight. Call mobile_testing_agent_begin first."); + } + return session; + } + + /** + * Starts a recording against whatever the panel has selected. + * + * It never picks for the user. Choosing the project is an explicit act — the panel's picker is + * an OS folder dialog, which must not open inside an agent's turn — and the device is whichever + * one the mirror is showing. Missing either, the refusal says which, and the user fixes it in + * the panel where the choice lives. + */ + begin(input: { title: string; goal: string }) { + if (this.session?.status === "recording") { + refused(`An agent recording is already in flight: "${this.session.title}". Finish it with mobile_testing_agent_finish, or drop it with mobile_testing_agent_abort.`); + } + const context = this.context; + const missing = [ + context?.projectPath ? "" : "an e2e project", + context?.deviceId ? "" : "a booted simulator or emulator in the mirror", + context?.bundleId ? "" : "a target with a bundle id in the project's e2e config", + ].filter(Boolean); + if (!context || missing.length > 0) { + refused(`The Testing panel has no ${missing.join(", no ")} selected, so there is nothing to record against. Ask the user to open Mobile Dev's Testing tab and choose one, then begin again.`); + } + this.session = { + target: { deviceId: context.deviceId, platform: context.platform, projectPath: context.projectPath, bundleId: context.bundleId }, + title: input.title.trim() || "a recorded flow", + goal: input.goal.trim(), + steps: [], status: "recording", note: "", error: "", draftId: "", + readId: 0, refs: new Map(), components: [], rows: [], truncated: 0, readAt: 0, identities: new Set(), + }; + this.changed(); + return { + ok: true, + title: this.session.title, + projectPath: context.projectPath, + deviceId: context.deviceId, + platform: context.platform, + bundleId: context.bundleId, + next: "Call mobile_testing_agent_screen to read the screen, then mobile_testing_agent_step with one of its refs. Re-read after every step that acts.", + }; + } + + /** + * The runner's element tree as the agent's eyes. + * + * One row per element, carrying what it is and — the part that matters — **the selector the + * recorder would emit for it**, with its rank and whether it is unique. The agent never writes a + * locator: it names a `ref` and the selector is resolved here, by the same `resolveChain` a tap + * on the mirror runs, so an agent-recorded step and a hand-recorded one are the same step. + */ + async screen() { + const session = this.recording(); + // A read the step path already paid for answers this one, while it is still describing the + // screen it was taken of. The agent is told to re-read after every acting step and that read + // has already happened, so the loop costs one snapshot per step rather than two. + if (session.rows.length === 0 || performance.now() - session.readAt > TREE_TTL_MS) await this.refresh(session); + return { + readId: session.readId, + ...(session.screen ? { screen: session.screen } : {}), + elements: session.rows, + ...(session.truncated > 0 + ? { truncated: session.truncated, note: `${session.truncated} further elements were left out of this read. Scroll with a swipe step if what you need is not here.` } + : {}), + }; + } + + /** Reads the device and makes that read the session's current tree, refs and rows. */ + private async refresh(session: Session): Promise<void> { + const startedAt = performance.now(); + let tree: Awaited<ReturnType<TestingDevices["read"]>>; + try { + tree = await this.devices.read(session.target); + } catch (error) { + // The refs the agent holds described a screen this read failed to replace; none of them can + // be trusted now, so a step on one is refused rather than acted on. + session.refs = new Map(); + session.rows = []; + recordTestingAgentRead("failed", performance.now() - startedAt, 0, session.target.platform); + throw error; + } + session.readId += 1; + session.components = tree.tree; + session.screen = tree.screen; + session.refs = new Map(); + session.identities = new Set(tree.tree.map(identity)); + // Off-screen elements stay in the read, flagged. The manual recorder never had to think about + // them — the user points at what they can see — but an agent asserting the *result* of a + // filter is often asserting a row below the fold, and `getByText` resolves it whether or not + // it is on screen. Dropping them hid exactly what the test was being written about. + session.rows = tree.tree.slice(0, MAX_ELEMENTS).map(component => { + const ref = `${session.readId}-${component.nodeId ?? ""}`; + session.refs.set(ref, { component, ...(component.nodeId && tree.refs[component.nodeId] ? { ref: tree.refs[component.nodeId] } : {}) }); + const resolved = resolveComponentSelector(tree.tree, component, tree.screen); + return { + ref, + ...(component.role ? { role: component.role } : {}), + ...(component.label ? { name: component.label } : {}), + ...(component.text && component.text !== component.label ? { text: component.text } : {}), + ...(component.placeholder ? { placeholder: component.placeholder } : {}), + ...(component.value ? { value: component.value } : {}), + ...(component.identifier ? { testId: component.identifier } : {}), + ...(component.disabled ? { enabled: false } : {}), + ...(component.selected ? { selected: true } : {}), + // A role query skips it and a tap cannot land on it; text and label queries still resolve + // it, so it is an element to assert about or to scroll to, not one to act on. + ...(component.hidden ? { offScreen: true } : {}), + bounds: [Math.round(component.bounds.x), Math.round(component.bounds.y), Math.round(component.bounds.width), Math.round(component.bounds.height)], + // Nothing here means nothing on this screen names the element: record it with + // `instruction` instead of a `ref`, which is the only time an agent step is the right step. + ...(resolved ? { + selector: selectorSource(resolved.selector), + rank: RANK_LABEL[resolved.selector.kind] ?? resolved.selector.kind, + unique: !resolved.ambiguous, + } : {}), + }; + }); + session.truncated = tree.tree.length - session.rows.length; + session.readAt = performance.now(); + recordTestingAgentRead("success", performance.now() - startedAt, tree.tree.length, session.target.platform); + } + + /** + * Records one step, and takes it when it is an action. + * + * A step recorded but never taken leaves the app on the screen before it, so the next step would + * be resolved against a screen the test will never reach — the rule the mirror's command menu + * already follows. An assertion reads and changes nothing, so it records without touching the + * device. + */ + async step(input: { ref?: string; kind?: string; value?: string; direction?: string; instruction?: string }) { + const session = this.recording(); + const kind = (input.kind ?? (input.instruction ? "agentAct" : "tap")) as PinnedStepKind | AgentStepKind; + const instruction = input.instruction?.trim(); + if ((AGENT_KINDS as readonly string[]).includes(kind)) { + if (!instruction) refused(`${kind} is a sentence the runner resolves at run time, so it needs an instruction.`); + return this.recordAgentStep(session, kind as AgentStepKind, instruction); + } + if (!(PINNED_KINDS as readonly string[]).includes(kind)) { + refused(`"${kind}" is not a step the runner has. Use one of: ${PINNED_KINDS.join(", ")}, or an instruction with agentAct/agentAssert/agentWaitFor.`); + } + if (instruction && !input.ref) return this.recordAgentStep(session, "agentAct", instruction); + return this.recordPinnedStep(session, kind as PinnedStepKind, input); + } + + private recordAgentStep(session: Session, kind: AgentStepKind, instruction: string) { + // No locator to resolve, so the tree is never read and nothing is sent to the device: taking + // an agent step would mean running a model against the live app mid-recording, which is a run. + const step: AgentStep = { kind, instruction, id: crypto.randomUUID(), label: agentStepLabel(kind, instruction) }; + session.steps.push(step); + this.changed(); + return { ok: true, index: session.steps.length, label: step.label, source: stepExpression(step), acted: false }; + } + + private async recordPinnedStep(session: Session, kind: PinnedStepKind, input: { ref?: string; value?: string; direction?: string }) { + const value = input.value; + if (stepNeedsValue(kind) && (value === undefined || value === "")) { + refused(`${kind} compares or types text, so it needs a value.`); + } + const direction = swipeDirection(input.direction); + // A swipe with no element is a swipe of the viewport, which is how a flow scrolls to something + // that is not on screen yet. It is the one pinned step that names nothing and still emits. + if (!input.ref) { + if (kind !== "swipe") refused(`${kind} acts on an element, so it needs a ref from mobile_testing_agent_screen. Only swipe can be taken without one, and it then swipes the viewport.`); + const centre = { x: (session.screen?.width ?? 0) / 2, y: (session.screen?.height ?? 0) / 2 }; + const anchor = anchorSelector(session.components, centre, session.screen); + return this.addStep(session, { + kind, selector: { kind: "point", point: centre }, direction, + // A swipe is as actionable as a tap: the viewport is ready from the first frame, so + // without something on the screen to wait for it fires into whatever is still loading. + ...(anchor && anchor.kind !== "point" ? { anchor } : {}), + id: crypto.randomUUID(), label: stepLabel("swipe", { kind: "point", point: centre }, direction), + // The runner performs `screen.swipe` as agent-device's own `scroll`, so this does too — + // same call, same direction, no trajectory of our own to fall outside a viewport that the + // keyboard has shrunk. + }, { kind: "scroll", direction }); + } + const entry = session.refs.get(input.ref); + if (!entry) { + refused(`${input.ref} is not from the latest screen read. Every action changes the screen, so call mobile_testing_agent_screen again and use a ref from that read.`); + } + const resolved = resolveComponentSelector(session.components, entry.component, session.screen); + if (!resolved) { + refused(`Nothing on this screen names ${input.ref} — no testID, no role with a name, no text of its own. Record it as a sentence instead: mobile_testing_agent_step with an instruction and no ref.`); + } + const step: PinnedStep = { + id: crypto.randomUUID(), + kind, + selector: resolved.selector, + label: stepLabel(kind, resolved.selector, kind === "swipe" ? direction : value), + ...(kind === "swipe" ? { direction } : value === undefined ? {} : { value }), + ...(resolved.ambiguous ? { ambiguous: true as const } : {}), + // Kept so the panel's step list can offer the same commands long after this screen has gone. + target: stepTarget(resolved.component ?? entry.component), + }; + const effect = stepEffect(kind); + // A swipe is scoped by the element's own box rather than placed on it, so it is the one acting + // step that does not need a handle. The rest are placed on the element, which is what makes + // the tap land where the runner's locator action will land. + if (effect === "swipe") return this.addStep(session, step, { kind: "swipe", ...swipePath(entry.component.bounds, direction) }); + if (effect === "none") return this.addStep(session, step); + const deviceRef = entry.ref; + if (!deviceRef) { + refused(`${input.ref} has no handle the device can act on, so this step cannot be taken. Record it as an assertion, or use a sentence with an instruction.`); + } + return this.addStep(session, step, effect === "type" + ? { kind: "type", ref: deviceRef, text: value ?? "" } + : { kind: effect === "doubleTap" ? "doubleTap" : "tap", ref: deviceRef }); + } + + private async addStep(session: Session, step: PinnedStep, action?: Parameters<TestingDevices["perform"]>[1]) { + session.steps.push(step); + this.changed(); + const result = { ok: true, index: session.steps.length, label: step.label, source: stepExpression(step) }; + if (!action) return { ...result, acted: false }; + const before = session.identities; + try { + await this.devices.perform(session.target, action); + } catch (error) { + // The step is recorded and the screen is not where the step says it is, which is worse than + // either on its own: say so rather than letting the next read be about a screen the flow + // never reached. + session.error = errorMessage(error); + this.changed(); + // Every action invalidates the tree, so the refs from the last read go with it. + session.refs = new Map(); + session.rows = []; + throw error; + } + // Reading here rather than leaving it to the agent's next `agent_screen` is what makes the + // navigation hint possible at all — the comparison needs both sides — and it costs nothing, + // because that read is the one `agent_screen` then serves. + await this.refresh(session); + const navigated = hasNavigated(before, session.identities); + return { + ...result, acted: true, + // Advisory, never enforcement. Plenty of real flows cross screens on purpose — a checkout, a + // sign-in — and a recorder that refused them would be worse than one that occasionally + // records a step too many. So this says what happened and what the recording was for, and + // leaves the judgement where the reasoning is. + ...(navigated ? { + navigated: true, + goal: session.goal, + next: `That step replaced the screen. This recording is for: ${session.goal || session.title}. If the screen you have arrived at is outside that, assert that you arrived and call mobile_testing_agent_finish; only keep recording if the flow the user asked for genuinely crosses screens.`, + } : { next: "The screen has changed. Call mobile_testing_agent_screen before the next step." }), + }; + } + + /** + * Ends the recording with the steps left dirty in the panel. + * + * Nothing is saved and nothing is run. The user adjusts the flow, names the file and saves it + * themselves — a run would also reset the app and take the device session, which is the one + * thing the panel's recorder still needs. + */ + async finish(input: { note?: string }) { + const session = this.recording(); + session.status = "finished"; + session.note = input.note?.trim() ?? ""; + await this.keep(session); + this.changed(); + return { + ok: true, + steps: session.steps.length, + source: emitE2eTest(session.steps, { title: session.title }), + next: `${session.steps.length} step${session.steps.length === 1 ? "" : "s"} are waiting in Mobile Dev's Testing panel, unsaved. Tell the user to review them there and press Save; nothing has been written to the project and nothing has been run.`, + }; + } + + /** Drops the session. Whatever was recorded stays in the panel, which is the user's to keep. */ + async abort(input: { reason: string }) { + const session = this.session; + if (!session || session.status !== "recording") return { ok: true, steps: session?.steps.length ?? 0, wasRecording: false }; + session.status = "aborted"; + session.note = input.reason.trim(); + await this.keep(session); + this.changed(); + return { ok: true, steps: session.steps.length, wasRecording: true }; + } + + /** + * Keeps a finished session's steps as an unsaved recording. + * + * Best effort on purpose: the agent's turn ends with the flow in the panel, and a draft that + * could not be written must not turn a recording that worked into a tool error. The panel still + * has the steps either way — it polls `state()` — and its own autosave is the second chance. + */ + private async keep(session: Session): Promise<void> { + if (session.steps.length === 0) return; + try { + const draft = await this.drafts.save({ + projectPath: session.target.projectPath, + title: session.title, + origin: "agent", + steps: session.steps, + }); + session.draftId = draft.id; + } catch (error) { + captureServerError(error, "testing.agent.draft"); + } + } +} + +const read = { readOnlyHint: true, destructiveHint: false, openWorldHint: false }; +const write = { readOnlyHint: false, destructiveHint: false, openWorldHint: false }; +/** The panel's own plumbing; the model drives the recording through the five tools below. */ +const appVisibility = { ui: { visibility: ["app"] as ("app" | "model")[] } }; +/** + * Visible to the model *and* to the panel, which is the whole point of this file: the agent drives + * the recording and the panel shows it. The eight tools in `testing-tools.ts` stay app-only — + * writing a file and running a test are the user's acts, not a step in an agent's loop. + */ +const sharedVisibility = { ui: { visibility: ["app", "model"] as ("app" | "model")[] } }; + +function safe<T>(handler: (input: T) => Promise<CallToolResult> | CallToolResult) { + return async (input: T): Promise<CallToolResult> => { + try { + return await handler(input); + } catch (error) { + captureServerError(error, "testing.agent"); + return { isError: true, content: [{ type: "text", text: errorMessage(error) }] }; + } + }; +} + +function dataResult(data: Record<string, unknown>): CallToolResult { + return { content: [{ type: "text", text: JSON.stringify(data) }], structuredContent: data }; +} + +/** Registers the agent recording tools beside `registerTestingTools`, on the same device session. */ +export function registerTestingAgentTools(server: McpServer, devices: TestingDevices, recordings = new TestingAgentRecordings(devices)): TestingAgentRecordings { + registerAppTool(server, "mobile_testing_agent_begin", { + title: "Begin recording an e2e test", + description: + "Start recording an e2e test into Mobile Dev's Testing panel, against the project, device and app the panel already has selected. Refuses, saying what is missing, when the user has not chosen them. The steps appear in the panel as you record them and are never saved or run by you.", + inputSchema: { + title: z.string().min(1).max(200).describe("The test's title, e.g. \"the explore details screen\"."), + goal: z.string().min(1).max(500).describe("What this flow covers, in one sentence. Shown to the user in the panel's banner."), + }, + annotations: write, + _meta: sharedVisibility, + }, safe(async (input: { title: string; goal: string }) => dataResult(recordings.begin(input)))); + + registerAppTool(server, "mobile_testing_agent_screen", { + title: "Read the screen being recorded", + description: + "Read the element tree of the screen the recording is on: one row per element with its role, name, text, placeholder, value, testId, state and bounds, and the selector a step on it would emit, its rank, and whether it is unique. Name a ref from this read in mobile_testing_agent_step; never write a locator yourself. Every acting step changes the screen, so read again after each one.", + inputSchema: {}, + annotations: read, + _meta: sharedVisibility, + }, safe(async () => dataResult(await recordings.screen()))); + + registerAppTool(server, "mobile_testing_agent_step", { + title: "Record one step", + description: + "Record one step of the flow and, when it acts, take it on the device so the next step is read from the screen the test will be on. Pass a ref from the last screen read; assertions record without touching the device. Pass an instruction and no ref only for an element the screen read reported no selector for. The result says navigated: true when the step replaced the screen, with the recording's goal as a reminder of its scope.", + inputSchema: { + ref: z.string().min(1).max(64).optional().describe("An element ref from the latest mobile_testing_agent_screen."), + kind: z.enum([...PINNED_KINDS, ...AGENT_KINDS]).optional().describe("The step; defaults to tap, or agentAct with an instruction."), + value: z.string().max(2048).optional().describe("The text fill types, or the text an assertion compares with."), + direction: z.enum(["up", "down", "left", "right"]).optional().describe("Which way a swipe goes. A swipe with no ref swipes the viewport."), + instruction: z.string().max(1000).optional().describe("A sentence for agent.act/agent.assert, for an element nothing can name."), + }, + annotations: write, + _meta: sharedVisibility, + }, safe(async (input: { ref?: string; kind?: string; value?: string; direction?: string; instruction?: string }) => + dataResult(await recordings.step(input)))); + + registerAppTool(server, "mobile_testing_agent_finish", { + title: "Finish the recording", + description: + "End the recording. The steps are left unsaved in the Testing panel for the user to adjust, save and run; nothing is written to the project and nothing is run.", + inputSchema: { note: z.string().max(500).optional().describe("One line on what was recorded, for the user.") }, + annotations: write, + _meta: sharedVisibility, + }, safe(async (input: { note?: string }) => dataResult(await recordings.finish(input)))); + + registerAppTool(server, "mobile_testing_agent_abort", { + title: "Abandon the recording", + description: "Drop the recording. Whatever was recorded stays in the Testing panel for the user.", + inputSchema: { reason: z.string().min(1).max(500).describe("Why the recording stopped.") }, + annotations: write, + _meta: sharedVisibility, + }, safe(async (input: { reason: string }) => dataResult(await recordings.abort(input)))); + + registerAppTool(server, "mobile_testing_agent_state", { + title: "Read the agent recording", + description: "Read the agent recording's status and steps. The Testing panel polls this and adopts the steps as they arrive.", + inputSchema: {}, + annotations: read, + _meta: appVisibility, + }, safe(() => dataResult({ ...recordings.state() }))); + + registerAppTool(server, "mobile_testing_set_context", { + title: "Report the Testing panel's selection", + description: + "Tell the server which project, device and bundle id the Testing panel has selected, so an agent recording has something to record against. The panel calls this whenever the selection changes; nothing is ever inferred from the server's own working directory.", + inputSchema: { + projectPath: z.string().max(4096), + deviceId: deviceIdSchema.or(z.literal("")), + platform: z.enum(["ios", "android"]).optional(), + bundleId: z.string().max(512), + }, + annotations: write, + _meta: appVisibility, + }, safe((input: Omit<TestingAgentContext, "platform"> & { platform?: "ios" | "android" }) => { + if (input.projectPath && input.deviceId && input.platform && input.bundleId) recordings.setContext({ ...input, platform: input.platform }); + else recordings.clearContext(); + return dataResult({ ok: true }); + })); + + return recordings; +} diff --git a/packages/core/src/testing-device.ts b/packages/core/src/testing-device.ts new file mode 100644 index 00000000..e5f5140e --- /dev/null +++ b/packages/core/src/testing-device.ts @@ -0,0 +1,367 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; +import type { ScreenComponent, ScreenSize } from "@mobile-dev/shared/screen-annotations.ts"; +import { APP_FLOW_DIRECTORY, loadVendoredAgentDevice } from "./app-flow-host.ts"; +import { captureServerError } from "./telemetry.ts"; + +/** + * The element tree the Testing panel's recorder names selectors from: the one the `e2e` runner + * will query when it runs the recorded test. + * + * The recorder used to read the macOS accessibility API through Baguette and translate its role + * vocabulary into the runner's with a hand-maintained table. Two trees over the same pixels never + * agree for long — iOS reports a tab bar item as a radio button, synthesises `AXLink` from the + * link *trait* that any view can carry, and offers a search field's placeholder as its name — and + * every divergence surfaced as a recorded test that simply did not resolve. So the recorder reads + * the runner's tree instead: the same `agent-device` snapshot, projected by the project's own + * `@e2e-dev/mobile` build. Role vocabulary and name sources then agree by construction, because + * there is only one of each. + * + * Baguette still owns the mirror. Only the recorder's *read* path moves, and the two hold the + * same simulator at once: a snapshot taken while the mirror streams leaves the stream running and + * returns in about a tenth of a second. + */ + +/** One `SemanticNode` as `@e2e-dev/mobile` projects it; only the fields a selector is built from. */ +interface SemanticNode { + role?: string; + name?: string; + text?: string; + value?: string; + testId?: string; + attributes?: Record<string, string | undefined>; + states?: Record<string, boolean | undefined>; + rect?: { x: number; y: number; width: number; height: number }; +} +/** + * `ref` is agent-device's own handle for the node, which is what an action is placed on: the + * projection keeps it beside the semantic node, so a selector the panel resolved and a tap the + * server performs are about the same element without a second lookup. It is absent for a node + * agent-device did not give one. + */ +interface ProjectedNode { id: string; kind: string; ref?: string; node: SemanticNode; parent?: ProjectedNode } +interface ProjectedSnapshot { index: readonly ProjectedNode[]; viewport?: ScreenSize } +interface EngineProjection { + projectSnapshot(raw: readonly unknown[], options: { mintId: () => string }): ProjectedSnapshot; +} + +/** What one device's recording session needs to attach to the app the user is already tapping. */ +export interface TestingDeviceTarget { + /** An iOS simulator UDID or an Android emulator's adb serial. */ + deviceId: string; + platform: "ios" | "android"; + /** The e2e project whose `@e2e-dev/mobile` projects the snapshot; its build, not ours. */ + projectPath: string; + /** The app to attach to, from the project's config. Never launched fresh — see `attach`. */ + bundleId: string; +} + +export interface TestingDeviceTree { + tree: ScreenComponent[]; + screen?: ScreenSize; + /** + * Each element's agent-device ref, by the `nodeId` the panel knows it as. + * + * The panel never needs this — it records selectors and drives the mirror — but an agent + * recording acts through agent-device itself, and a `ref` is what its tap, fill and scroll take. + * Keeping the pair here is what stops the server resolving the element a second time, and a + * second resolution is a second answer. + */ + refs: Record<string, string>; +} + +/** + * One thing an agent recording does to the device, after the step has been recorded. + * + * The kinds are `stepEffect`'s, not `PinnedStepKind`'s: an assertion changes nothing, so it never + * reaches here, and `scrollUntilVisible` is only ever offered for an element that is on screen + * already. The same reduction the mirror's menu makes, made once in `test-recording.ts`. + */ +export type TestingDeviceAction = + | { kind: "tap"; ref: string } + | { kind: "doubleTap"; ref: string } + | { kind: "type"; ref: string; text: string } + // An element-scoped swipe, drawn inside the element's own box. + | { kind: "swipe"; from: { x: number; y: number }; to: { x: number; y: number } } + /** + * A swipe of the viewport, which the runner performs as `interactions.scroll` — so this does + * too. Deriving a trajectory from the snapshot's viewport instead is what agent-device refuses + * with "Gesture trajectory does not fit inside the viewport" whenever something shrinks the + * window the gesture runtime measures, the on-screen keyboard above all. + */ + | { kind: "scroll"; direction: "up" | "down" | "left" | "right" }; + +/** + * agent-device allows exactly one session per device, so a leaked session makes the user's next + * `e2e run` fail with a refusal they cannot connect to anything they did. One name, released on + * every path out of recording. + */ +const SESSION = "mobile-dev-recorder"; + +/** + * Pins every daemon this app starts to one state directory, so they are same-state-dir peers that + * can take the iOS runner lease from one another rather than strangers that cannot. App-flow pins + * the same directory for the same reason; whichever runs first wins and the other agrees. + */ +function pinStateDir(): void { + if (process.env.AGENT_DEVICE_STATE_DIR?.trim()) return; + process.env.AGENT_DEVICE_STATE_DIR = join(APP_FLOW_DIRECTORY, "agent-device"); +} + +/** + * The project's own node projection: the `ROLE_MAP`, the React Native role descriptions, and the + * name/placeholder sources the runner will use on this very snapshot. Loaded from the project + * rather than vendored here, because a copy is a second vocabulary again — the thing this whole + * path exists to remove. + */ +async function loadProjection(projectPath: string): Promise<EngineProjection> { + const entry = join(projectPath, "node_modules/@e2e-dev/mobile/dist/nodes.js"); + if (!existsSync(entry)) { + throw new ExpectedOperationError("invalid_input", + "This project has no @e2e-dev/mobile installed, so the recorder cannot read the tree its tests will run against. Install the project's dependencies, then record again."); + } + // Deep path rather than a package specifier: the package does not export this module, and a + // copy of its mapping here would be the divergence this path exists to remove. + const dynamicImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<unknown>; + return await dynamicImport(pathToFileURL(entry).href) as EngineProjection; +} + +function textOf(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +/** + * One projected node as the panel's flat component list. The names are the panel's; the values + * are the runner's, passed through untouched — `role` above all, which is why no table survives + * in `test-recording.ts`. + */ +function toComponent(entry: ProjectedNode, depth: number): ScreenComponent | undefined { + const rect = entry.node.rect; + if (!rect || !(rect.width > 0) || !(rect.height > 0)) return; + const name = textOf(entry.node.name); + const text = textOf(entry.node.text); + const testId = textOf(entry.node.testId); + const placeholder = textOf(entry.node.attributes?.["placeholder"]); + return { + name: name ?? text ?? testId ?? placeholder ?? entry.kind, + bounds: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, + depth, + source: "accessibility", + nodeId: entry.id, + ...(entry.parent ? { parentId: entry.parent.id } : {}), + ...(entry.node.role ? { role: entry.node.role } : {}), + ...(testId ? { identifier: testId } : {}), + // `label` is the runner's `name`: what `getByRole(role, name)` and `getByLabel` match. + ...(name ? { label: name } : {}), + ...(text ? { text } : {}), + ...(textOf(entry.node.value) ? { value: textOf(entry.node.value) } : {}), + ...(placeholder ? { placeholder } : {}), + // The runner skips a hidden node for a role query; `hidden` is set only by `visibleToUser: false`. + ...(entry.node.states?.["hidden"] === true ? { hidden: true } : {}), + // What `toBeDisabled` and `toBeSelected` read, so an agent reading the screen can tell a + // selected tab from the rest without a second call. + ...(entry.node.states?.["disabled"] === true ? { disabled: true } : {}), + ...(entry.node.states?.["selected"] === true ? { selected: true } : {}), + }; +} + +function depthOf(entry: ProjectedNode): number { + let depth = 0; + for (let parent = entry.parent; parent; parent = parent.parent) depth += 1; + return depth; +} + +export function projectedComponents(projection: EngineProjection, nodes: readonly unknown[]): TestingDeviceTree { + let minted = 0; + const snapshot = projection.projectSnapshot(nodes, { mintId: () => `n${++minted}` }); + const tree: ScreenComponent[] = []; + const refs: Record<string, string> = {}; + for (const entry of snapshot.index) { + const component = toComponent(entry, depthOf(entry)); + if (!component) continue; + tree.push(component); + // agent-device's own refs are `@e20`; the projection strips the `@` for its own use, and + // `interactions.click({ ref })` parses a bare `e20` as a selector and refuses it. So the + // sigil is put back here, at the one boundary that hands a ref to agent-device. + if (entry.ref) refs[entry.id] = entry.ref.startsWith("@") ? entry.ref : `@${entry.ref}`; + } + return { tree, refs, ...(snapshot.viewport ? { screen: snapshot.viewport } : {}) }; +} + +type AgentDeviceClient = { + apps: { open(options: Record<string, unknown>): Promise<unknown> }; + capture: { snapshot(options: Record<string, unknown>): Promise<{ nodes?: unknown }> }; + sessions: { close(options?: Record<string, unknown>): Promise<unknown> }; + interactions: { + click(options: Record<string, unknown>): Promise<unknown>; + fill(options: Record<string, unknown>): Promise<unknown>; + swipe(options: Record<string, unknown>): Promise<unknown>; + scroll(options: Record<string, unknown>): Promise<unknown>; + }; +}; + +/** The error agent-device raises when the session exists but nothing is attached to an app. */ +function needsAttach(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /active app session|SESSION_NOT_FOUND|no active session/i.test(message); +} + +/** + * Reads the runner's tree for the device the panel is mirroring, and owns the session that read + * needs. One session, one device, released explicitly — see `release`. + */ +export class TestingDevices { + private client?: Promise<AgentDeviceClient>; + private readonly projections = new Map<string, Promise<EngineProjection>>(); + /** The device an attached session is on, so a second read reuses it instead of reopening. */ + private attached?: string; + /** Serialises reads: agent-device answers one command per session at a time. */ + private queue: Promise<unknown> = Promise.resolve(); + + private readonly load: typeof loadVendoredAgentDevice; + + constructor(load = loadVendoredAgentDevice) { this.load = load; } + + private async connect(): Promise<AgentDeviceClient> { + pinStateDir(); + this.client ??= this.load().then(module => module.createAgentDeviceClient({}) as AgentDeviceClient); + return this.client; + } + + private projection(projectPath: string): Promise<EngineProjection> { + let pending = this.projections.get(projectPath); + if (!pending) { + pending = loadProjection(projectPath); + // A failed load must not be cached: the user installs the dependency and records again. + pending.catch(() => this.projections.delete(projectPath)); + this.projections.set(projectPath, pending); + } + return pending; + } + + /** + * Attaches to the app the user is already in. + * + * `relaunch: false` is the whole point: the recorder exists so the user keeps tapping the screen + * they are on, and a relaunch would throw away the state they navigated to. `--foreground` is + * never used — it refuses outright when more than one app is running on the simulator, which is + * every simulator anyone has been working on. + */ + private async attach(client: AgentDeviceClient, target: TestingDeviceTarget): Promise<void> { + await client.apps.open({ + ...this.selector(target), session: SESSION, + app: target.bundleId, relaunch: false, noRecord: true, timeoutMs: 120_000, + }); + this.attached = this.key(target); + } + + /** The runner's own snapshot call, with the runner's own options: a different tree is a different test. */ + private snapshot(client: AgentDeviceClient, target: TestingDeviceTarget): Promise<{ nodes?: unknown }> { + return client.capture.snapshot({ ...this.selector(target), session: SESSION, interactiveOnly: false }); + } + + /** The device selector agent-device takes: an adb serial on Android, a UDID on iOS. */ + private selector(target: TestingDeviceTarget) { + return target.platform === "android" + ? { platform: "android" as const, serial: target.deviceId } + : { platform: "ios" as const, udid: target.deviceId }; + } + + /** Every input that decides which app/session is attached and how its tree is projected. */ + private key(target: TestingDeviceTarget): string { + return `${target.platform}\u0000${target.deviceId}\u0000${target.projectPath}\u0000${target.bundleId}`; + } + + async read(target: TestingDeviceTarget): Promise<TestingDeviceTree> { + const run = this.queue.then(() => this.readOnce(target), () => this.readOnce(target)); + this.queue = run.catch(() => undefined); + return run; + } + + private async readOnce(target: TestingDeviceTarget): Promise<TestingDeviceTree> { + const [client, projection] = await Promise.all([this.connect(), this.projection(target.projectPath)]); + if (this.attached !== this.key(target)) await this.attach(client, target); + let snapshot: { nodes?: unknown }; + try { + snapshot = await this.snapshot(client, target); + } catch (error) { + // The session can go while the panel is idle — the app was closed, the runner was swept. One + // reattach, then the failure is real and the panel says so. + if (!needsAttach(error)) throw error; + this.attached = undefined; + await this.attach(client, target); + snapshot = await this.snapshot(client, target); + } + if (!Array.isArray(snapshot.nodes)) throw new Error("The device snapshot returned no nodes."); + return projectedComponents(projection, snapshot.nodes); + } + + /** + * Takes the step an agent recording just recorded, on the same session the read used. + * + * Not the panel's own input path: the server cannot reach the mirror's stream, and this is the + * better half of the trade anyway — agent-device places the action on the element's own ref, so + * the tap lands where the runner's locator action will land rather than on a coordinate someone + * computed from a rectangle. The mirror shows every change regardless, because the video stream + * and the snapshot hold the same simulator at once. + * + * Every action asks agent-device to **settle** afterwards. The step the agent records next is + * resolved against the screen this one produced, and a screen mid-transition answers with refs + * that are already stale — agent-device refuses the next action outright with "had no confirmed + * resting position when the transition window ran out". The wait belongs here, where the + * transition is, rather than as a sleep the caller guesses at. + */ + async perform(target: TestingDeviceTarget, action: TestingDeviceAction): Promise<void> { + const run = this.queue.then(() => this.performOnce(target, action), () => this.performOnce(target, action)); + this.queue = run.catch(() => undefined); + return run; + } + + private async performOnce(target: TestingDeviceTarget, action: TestingDeviceAction): Promise<void> { + const client = await this.connect(); + if (this.attached !== this.key(target)) await this.attach(client, target); + const base = { ...this.selector(target), session: SESSION, settle: true }; + switch (action.kind) { + case "tap": await client.interactions.click({ ...base, ref: action.ref }); return; + // agent-device takes the double tap as one gesture, so unlike the mirror — whose input path + // has a single tap in it and sends two — this is the gesture the emitted `doubleTap` names. + case "doubleTap": await client.interactions.click({ ...base, ref: action.ref, doubleTap: true }); return; + // The runner's `fill` replaces a field's contents, and so does agent-device's, so an agent + // recording leaves the screen showing exactly what the step will produce — the one place it + // is more faithful than the mirror, which can only append. + case "type": await client.interactions.fill({ ...base, ref: action.ref, text: action.text }); return; + // `SwipeOptions` has no settle: a swipe is a gesture over the viewport rather than an + // action on an element, so there is no element whose resting position the next step needs. + case "swipe": await client.interactions.swipe({ ...this.selector(target), session: SESSION, from: action.from, to: action.to }); return; + case "scroll": await client.interactions.scroll({ ...base, direction: action.direction }); return; + } + } + + /** + * Gives the device back. + * + * agent-device refuses a second session on a device that already has one, so a session left open + * here makes the user's next `npx e2e run` fail with `Device is already in use` — a worse, + * stranger failure than any this path fixes. Recording stops, the panel closes, the device + * changes: each of those releases. + */ + async release(): Promise<void> { + const run = this.queue.then(() => this.releaseOnce(), () => this.releaseOnce()); + this.queue = run.catch(() => undefined); + return run; + } + + private async releaseOnce(): Promise<void> { + if (this.attached === undefined || this.client === undefined) return; + this.attached = undefined; + try { + await (await this.client).sessions.close({ session: SESSION }); + } catch (error) { + // A session that is already gone is the state we wanted; anything else is worth knowing + // about but must not fail the user's stop. + captureServerError(error, "testing.release"); + } + } +} diff --git a/packages/core/src/testing-drafts.ts b/packages/core/src/testing-drafts.ts new file mode 100644 index 00000000..c0eae6cc --- /dev/null +++ b/packages/core/src/testing-drafts.ts @@ -0,0 +1,155 @@ +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; +import { draftSummary, testDraftIdSchema, testDraftSchema } from "@mobile-dev/shared/test-drafts.ts"; +import type { TestDraft, TestDraftSummary } from "@mobile-dev/shared/test-drafts.ts"; +import { TESTING_DIRECTORY } from "./testing-projects.ts"; + +export const DRAFTS_DIRECTORY = join(TESTING_DIRECTORY, "drafts"); + +/** + * How many unsaved recordings are kept. Past this the oldest goes: a draft is a safety net for the + * flow the user is working on, not an archive, and an unbounded folder of them would be a list + * nobody can read and a prompt nobody answers. + */ +const MAX_DRAFTS = 30; + +function normalizeProject(projectPath: string): string { + const trimmed = projectPath.length > 1 ? projectPath.replace(/\/+$/, "") : projectPath; + return trimmed.normalize("NFC"); +} + +function sameProject(left: string, right: string): boolean { + return normalizeProject(left).localeCompare(normalizeProject(right), "en", { sensitivity: "base" }) === 0; +} + +/** + * The recordings the Testing panel has not written into a project yet. + * + * One file per draft, named by its id, the same shape `RecordingStore` uses for performance + * recordings: a draft is read and replaced whole, and a file per draft means a delete is an + * `unlink` rather than a rewrite of everyone else's work. Recorded steps are stored as data so + * reopening a flow restores its controls. When the user edits the code preview, the source is + * stored too and takes precedence over the generated output. + */ +export class TestDraftStore { + readonly directory: string; + /** Serializes writes, so two autosaves racing cannot interleave a prune with a save. */ + private updates: Promise<unknown> = Promise.resolve(); + + constructor(directory = DRAFTS_DIRECTORY) { this.directory = directory; } + + private path(id: string): string { return join(this.directory, `${id}.json`); } + + private async writeFileAtomic(id: string, draft: TestDraft): Promise<void> { + await mkdir(this.directory, { recursive: true, mode: 0o700 }); + const target = this.path(id); + const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify(draft), { mode: 0o600 }); + await rename(temporary, target); + } finally { + await rm(temporary, { force: true }).catch(() => undefined); + } + } + + private async ids(): Promise<string[]> { + let entries; + try { entries = await readdir(this.directory, { withFileTypes: true }); } + catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return []; + throw error; + } + return entries + .filter(entry => entry.isFile() && entry.name.endsWith(".json")) + .map(entry => entry.name.slice(0, -5)) + .filter(id => testDraftIdSchema.safeParse(id).success); + } + + /** Reads one draft, or undefined when it is gone or no longer parses. */ + async read(id: string): Promise<TestDraft | undefined> { + const parsedId = testDraftIdSchema.safeParse(id); + if (!parsedId.success) return undefined; + const text = await readFile(this.path(parsedId.data), "utf8").catch(() => undefined); + if (text === undefined) return undefined; + let parsed: unknown; + try { parsed = JSON.parse(text); } catch { return undefined; } + const draft = testDraftSchema.safeParse(parsed); + // A draft written by a newer plugin, or one that was truncated, is skipped rather than thrown: + // it must not take the rest of the list down with it. + if (!draft.success || draft.data.id !== parsedId.data) return undefined; + return draft.data as TestDraft; + } + + /** + * The drafts of one project, newest first. Unreadable files are dropped from the list; a project + * path of "" lists every draft, which is what the panel shows before a project is chosen. + */ + async list(projectPath = ""): Promise<TestDraftSummary[]> { + const drafts = await Promise.all((await this.ids()).map(id => this.read(id))); + return drafts + .filter((draft): draft is TestDraft => draft !== undefined) + .filter(draft => !projectPath || sameProject(draft.projectPath, projectPath)) + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)) + .map(draftSummary); + } + + /** + * Writes one draft, creating it when `id` names nothing. Returns what was stored, so the caller + * learns the id the panel then autosaves under. + */ + async save(input: { + id?: string; projectPath: string; title: string; fileName?: string; + origin?: "user" | "agent"; steps: TestDraft["steps"]; source?: string; + }): Promise<TestDraft> { + const projectPath = normalizeProject(input.projectPath.trim()); + if (!projectPath) throw new ExpectedOperationError("invalid_input", "Choose the project folder."); + if (input.steps.length === 0 && !input.source) throw new ExpectedOperationError("invalid_input", "Record a step before saving a draft."); + const next = this.updates.catch(() => undefined).then(async () => { + const existing = input.id ? await this.read(input.id) : undefined; + const now = new Date().toISOString(); + const draft: TestDraft = { + schemaVersion: 1, + id: existing?.id ?? (testDraftIdSchema.safeParse(input.id).success ? input.id! : randomUUID()), + projectPath, + title: input.title.slice(0, 160), + fileName: (input.fileName ?? existing?.fileName ?? "").slice(0, 256), + origin: input.origin ?? existing?.origin ?? "user", + steps: input.steps, + ...(input.source === undefined ? {} : { source: input.source }), + createdAt: existing?.createdAt ?? now, + updatedAt: now, + }; + const validated = testDraftSchema.parse(draft) as TestDraft; + await this.writeFileAtomic(validated.id, validated); + await this.prune(validated.id); + return validated; + }); + this.updates = next; + return await next; + } + + /** Forgets one draft. Deleting one that is already gone is not an error. */ + async remove(id: string): Promise<boolean> { + const parsed = testDraftIdSchema.safeParse(id); + if (!parsed.success) return false; + const next = this.updates.catch(() => undefined).then(async () => { + const path = this.path(parsed.data); + const existed = await stat(path).then(entry => entry.isFile(), () => false); + await rm(path, { force: true }); + return existed; + }); + this.updates = next; + return await next; + } + + /** Drops the oldest drafts past `MAX_DRAFTS`, never the one just written. */ + private async prune(keep: string): Promise<void> { + const drafts = await this.list(); + for (const draft of drafts.slice(MAX_DRAFTS)) { + if (draft.id === keep) continue; + await rm(this.path(draft.id), { force: true }).catch(() => undefined); + } + } +} diff --git a/packages/core/src/testing-projects.ts b/packages/core/src/testing-projects.ts new file mode 100644 index 00000000..2afa2fce --- /dev/null +++ b/packages/core/src/testing-projects.ts @@ -0,0 +1,442 @@ +import { createHash, randomUUID } from "node:crypto"; +import { homedir, tmpdir } from "node:os"; +import { lstat, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; +import { chooseProjectFolder } from "./project-folder-dialog.ts"; +import { recordProjectDialog } from "./telemetry.ts"; + +/** + * The projects the Testing panel can record against: folders holding an `e2e.config.ts`. The user + * always picks one explicitly — the server's own cwd is never assumed to be a project, because the + * host launches the server from wherever it likes and a wrong guess would write a test file into a + * folder the user never chose. + * + * The list lives on the server, like App Flow's, because the panel document is rebuilt for each + * host session and only the server can open the system folder picker. + */ +export const TESTING_DIRECTORY = join(homedir(), "Library/Application Support/mobile-dev/testing"); + +const REGISTRY_FILE = "projects.json"; +const MAX_PROJECTS = 12; +const CONFIG_FILES = ["e2e.config.ts", "e2e.config.mts", "e2e.config.js", "e2e.config.mjs"] as const; + +export interface TestingProjectChoice { + path: string; + name: string; + lastPickedAt: string; +} + +/** One target read out of the project's e2e config, as far as a static read can see it. */ +export interface TestingTarget { + name: string; + platform?: string; + bundleId?: string; +} + +export interface TestingProjectConfig { + path: string; + name: string; + /** Absolute path of the config file that was read. */ + configPath: string; + targets: TestingTarget[]; +} + +interface Choice { path: string; lastPickedAt: string } +interface Registry { version: 1; projects: Choice[] } + +function normalize(projectPath: string): string { + const trimmed = projectPath.length > 1 ? projectPath.replace(/\/+$/, "") : projectPath; + return trimmed.normalize("NFC"); +} + +function same(left: string, right: string): boolean { + return normalize(left).localeCompare(normalize(right), "en", { sensitivity: "base" }) === 0; +} + +async function isDirectory(path: string): Promise<boolean> { + return await stat(path).then(entry => entry.isDirectory(), () => false); +} + +async function isFile(path: string): Promise<boolean> { + return await stat(path).then(entry => entry.isFile(), () => false); +} + +async function readJson(path: string): Promise<unknown> { + try { return JSON.parse(await readFile(path, "utf-8")); } + catch { return null; } +} + +function timestamp(value: unknown): string { + return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : ""; +} + +/** The e2e config file of a project folder, or undefined when it holds none. */ +export async function findConfig(projectPath: string): Promise<string | undefined> { + for (const file of CONFIG_FILES) { + const candidate = join(projectPath, file); + if (await isFile(candidate)) return candidate; + } + return undefined; +} + +/** + * Reads the targets out of an e2e config without executing it. + * + * The config is TypeScript that imports the project's own engines, so evaluating it properly would + * mean running the user's code in the server process. A static read is enough for what the panel + * needs — a target name, its platform, and the bundle id to launch — and anything it cannot see is + * left undefined for the panel to ask about once. + */ +export function parseTargets(source: string): TestingTarget[] { + const targets: TestingTarget[] = []; + const block = source.match(/targets\s*:\s*\[([\s\S]*?)\n\s*\]/); + const body = block?.[1]; + if (!body) return targets; + // Each entry starts at a `name:` and runs to the next one, so a bundleId is attributed to the + // target it sits in rather than to the first target in the list. + const names = [...body.matchAll(/name\s*:\s*["'`]([^"'`]+)["'`]/g)]; + for (const [index, match] of names.entries()) { + const start = match.index ?? 0; + const end = index + 1 < names.length ? names[index + 1].index ?? body.length : body.length; + const entry = body.slice(start, end); + const platform = entry.match(/platform\s*:\s*["'`]([^"'`]+)["'`]/)?.[1]; + const bundleId = entry.match(/bundleId\s*:\s*["'`]([^"'`]+)["'`]/)?.[1]; + targets.push({ name: match[1], ...(platform ? { platform } : {}), ...(bundleId ? { bundleId } : {}) }); + } + return targets; +} + +/** Reads one project's e2e config. Throws when the folder is not an e2e project. */ +export async function inspectProject(projectPath: string): Promise<TestingProjectConfig> { + const path = normalize(projectPath.trim()); + if (!isAbsolute(path)) throw new ExpectedOperationError("invalid_input", "Choose the project folder."); + if (!(await isDirectory(path))) throw new ExpectedOperationError("invalid_input", "That project folder no longer exists."); + const configPath = await findConfig(path); + if (!configPath) throw new ExpectedOperationError("invalid_input", "That folder has no e2e.config.ts. Run `npx e2e init` in it first."); + const source = await readFile(configPath, "utf-8").catch(() => ""); + return { path, name: basename(path) || path, configPath, targets: parseTargets(source) }; +} + +export class TestingProjectChoices { + readonly directory: string; + /** The system folder picker, injectable so a test never opens a dialog. */ + private readonly pick: () => Promise<string | undefined>; + private updates: Promise<unknown> = Promise.resolve(); + + constructor(directory: string, pick: () => Promise<string | undefined> = chooseProjectFolder) { + this.directory = directory; + this.pick = pick; + } + + private get path(): string { return join(this.directory, REGISTRY_FILE); } + + private async read(): Promise<Choice[]> { + const saved = await readJson(this.path) as Partial<Registry> | null; + const entries = Array.isArray(saved?.projects) ? saved.projects : []; + return entries + .filter((entry): entry is Choice => typeof entry?.path === "string") + .map(entry => ({ path: entry.path, lastPickedAt: timestamp(entry.lastPickedAt) })); + } + + private static clean(projects: readonly Choice[]): TestingProjectChoice[] { + const seen = new Set<string>(); + const unique: TestingProjectChoice[] = []; + for (const entry of [...projects].sort((left, right) => right.lastPickedAt.localeCompare(left.lastPickedAt))) { + if (!isAbsolute(entry.path)) continue; + const path = normalize(entry.path); + if (seen.has(path)) continue; + seen.add(path); + unique.push({ path, name: basename(path) || path, lastPickedAt: entry.lastPickedAt }); + } + return unique.slice(0, MAX_PROJECTS); + } + + private async write(projects: readonly Choice[]): Promise<void> { + const registry: Registry = { version: 1, projects: projects.map(entry => ({ path: normalize(entry.path), lastPickedAt: entry.lastPickedAt })) }; + await mkdir(this.directory, { recursive: true, mode: 0o700 }); + const temporary = `${this.path}.${process.pid}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify(registry, null, 2), "utf-8"); + await rename(temporary, this.path); + } finally { + await unlink(temporary).catch(() => undefined); + } + } + + private async change(changed: (known: Choice[]) => Choice[]): Promise<TestingProjectChoice[]> { + const next = this.updates.catch(() => undefined).then(async () => { + const known = TestingProjectChoices.clean(changed(await this.read())).map(entry => ({ path: entry.path, lastPickedAt: entry.lastPickedAt })); + await this.write(known); + return TestingProjectChoices.clean(known); + }); + this.updates = next; + return await next; + } + + /** Recorded folders that still exist and still hold an e2e config, newest first. */ + async list(): Promise<TestingProjectChoice[]> { + const projects = TestingProjectChoices.clean(await this.read()); + const available = await Promise.all(projects.map(async entry => ((await findConfig(entry.path)) ? entry : null))); + return available.filter((entry): entry is TestingProjectChoice => entry !== null); + } + + /** + * Records a chosen folder. A folder with no e2e config is refused here rather than at the first + * recording, so the panel's own list never offers one the recorder cannot write a test for. + */ + async record(projectPath: string): Promise<TestingProjectChoice> { + const { path } = await inspectProject(projectPath); + const stamp = new Date().toISOString(); + const projects = await this.change(known => [{ path, lastPickedAt: stamp }, ...known.filter(entry => !same(entry.path, path))]); + const recorded = projects.find(entry => same(entry.path, path)); + if (!recorded) throw new ExpectedOperationError("invalid_input", "That project folder no longer exists."); + return recorded; + } + + /** Opens the system folder picker and records what it returns. Undefined means the user cancelled. */ + async choose(): Promise<TestingProjectChoice | undefined> { + const startedAt = Date.now(); + try { + const path = await this.pick(); + if (!path) { + recordProjectDialog("cancelled", Date.now() - startedAt); + return undefined; + } + const project = await this.record(path); + recordProjectDialog("chosen", Date.now() - startedAt); + return project; + } catch (error) { + recordProjectDialog("failed", Date.now() - startedAt); + throw error; + } + } +} + +/** Where a recorded test may be written: the project's own `tests/` folder, nothing else. */ +const TESTS_DIRECTORY = "tests"; +const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; +const MAX_TEST_BYTES = 256 * 1024; + +export interface WrittenTest { path: string; written: boolean; existed: boolean } + +/** + * Resolves the absolute path a recorded test may be written to. + * + * The panel can ask the server to write a file, so this is the boundary that decides where. A + * path is accepted only when it is inside the chosen project's `tests/` folder after resolution, + * which is what rejects `..`, an absolute path, and a name that climbs out through a symlinked + * directory. The caller has already been told the file must be a `.e2e.ts`. + */ +export async function resolveTestPath(projectPath: string, relativePath: string): Promise<string> { + const root = normalize(projectPath.trim()); + if (!isAbsolute(root)) throw new ExpectedOperationError("invalid_input", "Choose the project folder."); + const requested = relativePath.trim().replace(/\\/g, "/"); + const segments = requested.split("/").filter(Boolean); + if (segments[0] !== TESTS_DIRECTORY) throw new ExpectedOperationError("invalid_input", "A recorded test is written inside the project's tests/ folder."); + if (segments.length < 2) throw new ExpectedOperationError("invalid_input", "Name the test file."); + if (!segments.every(segment => SEGMENT.test(segment))) throw new ExpectedOperationError("invalid_input", "Use letters, digits, dots, dashes and underscores in the file name."); + if (!segments.at(-1)!.endsWith(".e2e.ts")) throw new ExpectedOperationError("invalid_input", "A recorded test file ends in .e2e.ts."); + + const testsRoot = resolve(root, TESTS_DIRECTORY); + const target = resolve(root, ...segments); + const inside = relative(testsRoot, target); + if (inside.startsWith("..") || isAbsolute(inside)) throw new ExpectedOperationError("invalid_input", "A recorded test is written inside the project's tests/ folder."); + + // A symlinked directory on the way down would land the write outside the project however the + // path reads, so each level from tests/ downwards is checked rather than trusting the resolved + // string. Levels that do not exist yet are created by the write and cannot be links. + let directory = testsRoot; + for (const segment of [undefined, ...segments.slice(1, -1)]) { + if (segment !== undefined) directory = join(directory, segment); + const info = await lstat(directory).catch(() => undefined); + if (info?.isSymbolicLink()) throw new ExpectedOperationError("invalid_input", "That tests folder is a symbolic link."); + } + const existing = await lstat(target).catch(() => undefined); + if (existing?.isSymbolicLink()) throw new ExpectedOperationError("invalid_input", "That test file is a symbolic link."); + if (existing && !existing.isFile()) throw new ExpectedOperationError("invalid_input", "That path is not a file."); + return target; +} + +/** + * Resolves the paths an app-requested run may hand to the project's CLI. + * + * Runs have the same boundary as reads and writes: the test must be below `tests/`, and the only + * config that may be executed is the config discovered in the selected project. Keeping this at + * the tool boundary prevents a crafted app call from turning the runner into an arbitrary module + * loader while leaving `runRecordedTest` independently usable by its lower-level tests. + */ +export async function resolveTestRunPaths(projectPath: string, relativePath: string, configPath?: string): Promise<{ + projectPath: string; + relativePath: string; + configPath?: string; +}> { + const project = await inspectProject(projectPath); + const testPath = await resolveTestPath(project.path, relativePath); + if (!(await isFile(testPath))) throw new ExpectedOperationError("invalid_input", "That test file is no longer there."); + if (configPath !== undefined) { + const requestedConfig = normalize(configPath.trim()); + if (!isAbsolute(requestedConfig) || !same(resolve(requestedConfig), resolve(project.configPath))) + throw new ExpectedOperationError("invalid_input", "Run the project with its own e2e config."); + } + return { + projectPath: project.path, + relativePath: relative(project.path, testPath).split(/[\\/]/).join("/"), + ...(configPath === undefined ? {} : { configPath: project.configPath }), + }; +} + +/** + * Writes one recorded test into the project. Without `overwrite` an existing file is reported + * rather than replaced, so the panel can ask before it destroys someone's test. + */ +export async function writeTestFile(projectPath: string, relativePath: string, contents: string, overwrite = false): Promise<WrittenTest> { + const { path: root } = await inspectProject(projectPath); + if (Buffer.byteLength(contents, "utf-8") > MAX_TEST_BYTES) throw new ExpectedOperationError("invalid_input", "That test is too large to write."); + const target = await resolveTestPath(root, relativePath); + const existed = await isFile(target); + if (existed && !overwrite) return { path: target, written: false, existed }; + await mkdir(dirname(target), { recursive: true }); + const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, contents, "utf-8"); + await rename(temporary, target); + } finally { + await unlink(temporary).catch(() => undefined); + } + return { path: target, written: true, existed }; +} + +/** One test the project already holds, as a row of the panel's list. */ +export interface StoredTest { + /** Path under the project, e.g. `tests/a-recorded-flow.e2e.ts`. */ + relativePath: string; + /** The file's own name, which is what the row reads as. */ + name: string; + updatedAt: string; + sizeBytes: number; +} + +/** How deep under `tests/` the listing walks. Deeper than a folder per feature is not a list. */ +const MAX_TEST_DEPTH = 3; +/** A project with more tests than this is browsed in an editor, not in a popover. */ +const MAX_LISTED_TESTS = 200; + +/** + * The `.e2e.ts` files already in the project's `tests/` folder, newest first. + * + * This is the other half of the panel's list: a recording that has been written into the project + * is no longer a draft, and before this it simply left the list when it was saved — the user + * pressed Save and watched their test disappear. Read rather than remembered, so a test written by + * hand, by an agent or by another checkout is in the list too. + * + * A project with no `tests/` folder yet is an empty list, not an error: nothing has been saved. + */ +export async function listTestFiles(projectPath: string): Promise<StoredTest[]> { + const { path: root } = await inspectProject(projectPath); + const testsRoot = resolve(root, TESTS_DIRECTORY); + const found: StoredTest[] = []; + const walk = async (directory: string, depth: number): Promise<void> => { + if (depth > MAX_TEST_DEPTH || found.length >= MAX_LISTED_TESTS) return; + const entries = await readdir(directory, { withFileTypes: true }).catch(() => []); + for (const entry of entries) { + if (found.length >= MAX_LISTED_TESTS) return; + const child = join(directory, entry.name); + // Symlinks are skipped rather than followed: the listing must not walk out of the project. + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) { await walk(child, depth + 1); continue; } + if (!entry.isFile() || !entry.name.endsWith(".e2e.ts")) continue; + const info = await stat(child).catch(() => undefined); + if (!info) continue; + found.push({ + relativePath: [TESTS_DIRECTORY, ...relative(testsRoot, child).split(/[\\/]/).filter(Boolean)].join("/"), + name: entry.name, + updatedAt: info.mtime.toISOString(), + sizeBytes: info.size, + }); + } + }; + await walk(testsRoot, 1); + return found.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); +} + +/** Reads a listed test for the panel's read-only source preview. */ +export async function readTestFile(projectPath: string, relativePath: string): Promise<string> { + const { path: root } = await inspectProject(projectPath); + const target = await resolveTestPath(root, relativePath); + const info = await stat(target).catch(() => undefined); + if (!info?.isFile()) throw new ExpectedOperationError("invalid_input", "That test file is no longer there."); + if (info.size > MAX_TEST_BYTES) throw new ExpectedOperationError("invalid_input", "That test is too large to preview."); + return readFile(target, "utf-8"); +} + +type StagedTest = { projectPath: string; relativePath: string; originalHash: string; source: string }; +const STAGED_TESTS_DIRECTORY = join(tmpdir(), "mobile-dev-testing-edits"); + +function sourceHash(source: string): string { return createHash("sha256").update(source).digest("hex"); } +function stagedPath(projectPath: string, relativePath: string): string { + return join(STAGED_TESTS_DIRECTORY, `${sourceHash(`${projectPath}\0${relativePath}`)}.json`); +} + +async function stagedTest(projectPath: string, relativePath: string): Promise<StagedTest | undefined> { + const source = await readFile(stagedPath(projectPath, relativePath), "utf-8").catch(() => undefined); + if (!source) return; + try { + const staged = JSON.parse(source) as StagedTest; + if (staged.projectPath === projectPath && staged.relativePath === relativePath + && /^[a-f0-9]{64}$/.test(staged.originalHash) && typeof staged.source === "string") return staged; + } catch { /* An incomplete temp file is not a test. */ } +} + +/** Opens the original plus any working copy left by an earlier panel session. */ +export async function previewTestFile(projectPath: string, relativePath: string) { + const original = await readTestFile(projectPath, relativePath); + const staged = await stagedTest(projectPath, relativePath); + return { source: staged?.source ?? original, originalHash: staged?.originalHash ?? sourceHash(original), staged: Boolean(staged) }; +} + +/** Replaces only the temporary copy. The project file stays unchanged until Save. */ +export async function stageTestFile(projectPath: string, relativePath: string, originalHash: string, source: string): Promise<void> { + await readTestFile(projectPath, relativePath); + if (!/^[a-f0-9]{64}$/.test(originalHash)) throw new ExpectedOperationError("invalid_input", "Open the test before editing it."); + if (Buffer.byteLength(source, "utf-8") > MAX_TEST_BYTES) throw new ExpectedOperationError("invalid_input", "That test is too large to save."); + const path = stagedPath(projectPath, relativePath); + const existing = await stagedTest(projectPath, relativePath); + if (existing && existing.originalHash !== originalHash) throw new ExpectedOperationError("invalid_input", "The working copy changed. Reopen the test before editing it."); + await mkdir(STAGED_TESTS_DIRECTORY, { recursive: true, mode: 0o700 }); + const temporary = `${path}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify({ projectPath, relativePath, originalHash, source } satisfies StagedTest), { mode: 0o600 }); + await rename(temporary, path); + } finally { await unlink(temporary).catch(() => undefined); } +} + +/** Commits the working copy only if nobody changed the project file after editing began. */ +export async function saveStagedTestFile(projectPath: string, relativePath: string): Promise<{ path: string; originalHash: string }> { + const staged = await stagedTest(projectPath, relativePath); + if (!staged) throw new ExpectedOperationError("invalid_input", "There are no staged edits for this test."); + const original = await readTestFile(projectPath, relativePath); + if (sourceHash(original) !== staged.originalHash) + throw new ExpectedOperationError("invalid_input", "The test changed on disk since it was opened. Reopen it before saving."); + const written = await writeTestFile(projectPath, relativePath, staged.source, true); + await unlink(stagedPath(projectPath, relativePath)); + return { path: written.path, originalHash: sourceHash(staged.source) }; +} + +/** Undo drops the working copy and returns the original source. */ +export async function discardStagedTestFile(projectPath: string, relativePath: string): Promise<{ source: string; originalHash: string }> { + const original = await readTestFile(projectPath, relativePath); + await unlink(stagedPath(projectPath, relativePath)).catch(() => undefined); + return { source: original, originalHash: sourceHash(original) }; +} + +/** + * Deletes one test the project holds. The panel asks before this runs, and the host asks again: + * this removes a file from the user's repository, which nothing here can put back. + */ +export async function deleteTestFile(projectPath: string, relativePath: string): Promise<{ path: string; removed: boolean }> { + const { path: root } = await inspectProject(projectPath); + const target = await resolveTestPath(root, relativePath); + const removed = await unlink(target).then(() => true, () => false); + return { path: target, removed }; +} diff --git a/packages/core/src/testing-run.ts b/packages/core/src/testing-run.ts new file mode 100644 index 00000000..85735c38 --- /dev/null +++ b/packages/core/src/testing-run.ts @@ -0,0 +1,426 @@ +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { ExpectedOperationError } from "@mobile-dev/shared/error-reporting.ts"; + +/** + * Runs one recorded test with the project's own `e2e` CLI, live per-step, and turns its report into + * per-line status. + * + * None of the CLI's four reporters gives live per-step output: `list` prints a permanent line only + * for *agent* steps and repaints deterministic ones inside a TTY window, `json` prints the whole + * document once at the end, and `report.json` is written from `onRunFinished`. But the runner's own + * event spine does carry per-step `start` and `end`, and `e2e.config.ts` takes **reporter objects** + * with an `onEvent` handler — so a consumer can have the events without any change to the runner. + * + * So the run is given a generated config that spreads the project's own and appends one reporter + * whose `onEvent` writes each `step` event to stderr as a tagged line. The config is written into + * the project's `.e2e/` — the runner's own scratch directory, and the only place from which `e2e` + * and `@e2e-dev/mobile` resolve — and removed when the run ends. It adds a reporter; it changes + * nothing else, and `--reporter` replaces only the built-in reporters, so the normal output stands. + * + * A `step` event carries no source position; only the finished report does. It carries + * `identity.stepIndex`, and the runner reports one step per awaited call in order, so the caller + * maps the index to a line through `EmittedTest.calls` and the report reconciles it at the end. + */ + +/** One step of the run, as the panel shows it against a line of the preview. */ +export interface TestRunStep { + /** The 1-based line in the test file the step came from. */ + line: number; + /** The runner's own name for the call, e.g. `locator.tap`. */ + api: string; + label: string; + status: string; + durationMs?: number; + error?: string; +} + +/** One live `step` event, flattened to what a line mark needs. */ +export interface TestRunProgress { + /** The step's position in its test, which is its position in `EmittedTest.calls`. */ + stepIndex: number; + phase: "start" | "end"; + api: string; + label: string; + /** Only on `end`. */ + status?: string; + error?: string; +} + +export interface TestRunResult { + /** The runner's own verdict: `passed`, `failed`, `timedOut`, … */ + status: string; + exitCode: number; + steps: TestRunStep[]; + /** How long the run itself took, in milliseconds. See `reportDuration`. */ + durationMs?: number; + /** Set when the report could not be read at all, so the panel can say why rather than go quiet. */ + output?: string; +} + +function record(value: unknown): Record<string, unknown> | undefined { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined; +} +function list(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } +function str(value: unknown): string { return typeof value === "string" ? value : ""; } + +/** The last stretch of output, for the case where there is no report to explain the exit code. */ +function tail(text: string, limit = 4000): string { + const trimmed = text.trim(); + return trimmed.length > limit ? `…${trimmed.slice(-limit)}` : trimmed; +} + +/** + * The runner's error shapes carry a message somewhere; the panel only ever shows one line of it. + * A step whose error cannot be read still reports its status, which is the part a mark depends on. + */ +function errorMessageOf(value: unknown): string | undefined { + const error = record(value); + if (!error) return; + const message = str(error.message) || str(error.value) || str(error.name); + return message ? message.split("\n")[0]!.slice(0, 400) : undefined; +} + +/** The steps of the attempt that decided each result — the last one, after any retries. */ +export function reportSteps(report: unknown, file: string): TestRunStep[] { + const run = record(record(report)?.run); + const steps: TestRunStep[] = []; + for (const entry of list(run?.results)) { + const result = record(entry); + if (!result) continue; + const attempts = list(result.attempts); + const attempt = record(attempts[attempts.length - 1]); + for (const item of list(attempt?.steps)) { + const step = record(item); + const source = record(step?.source); + const line = typeof source?.line === "number" ? source.line : 0; + // A step the runner attributes to another file is another test's; `app.open` has no line. + if (!step || line <= 0 || (str(source?.file) && str(source?.file) !== file)) continue; + const error = errorMessageOf(step.error); + steps.push({ + line, api: str(step.api), label: str(step.label), status: str(step.status) || "unknown", + ...(typeof step.durationMs === "number" ? { durationMs: step.durationMs } : {}), + ...(error ? { error } : {}), + }); + } + } + return steps; +} + +/** + * How long the run took, from the report itself rather than from the panel's own clock. + * + * The report has no run-level duration: `run.startedAt` is required and `run.finishedAt` is + * optional, so the span between them is the number when both are there. When the run was cut short + * — interrupted, or a reporter that never saw the end — there is no `finishedAt`, and the attempts + * are what is left: `attempt.durationMs` is required of every one, so their sum is the time the + * runner actually spent. It is the lower bound of the two, which is the honest way round: it + * undercounts the runner's own setup rather than inventing a span from a clock nobody recorded. + * + * This is deliberately not the panel's `ui.testing.run` timing, which brackets the whole call + * including spawning the CLI and resolving the project. The two answer different questions and + * neither should be read as the other. + */ +export function reportDuration(report: unknown): number | undefined { + const run = record(record(report)?.run); + if (!run) return; + const started = Date.parse(str(run.startedAt)); + const finished = Date.parse(str(run.finishedAt)); + if (Number.isFinite(started) && Number.isFinite(finished) && finished >= started) return finished - started; + let total = 0; + let seen = false; + for (const entry of list(run.results)) { + for (const item of list(record(entry)?.attempts)) { + const duration = record(item)?.durationMs; + if (typeof duration !== "number" || !Number.isFinite(duration)) continue; + total += duration; + seen = true; + } + } + return seen ? total : undefined; +} + +export function reportStatus(report: unknown, exitCode: number): string { + const run = record(record(report)?.run); + return str(run?.status) || (exitCode === 0 ? "passed" : "failed"); +} + +/** + * The `json` reporter prints the report document and nothing else, but a package manager, a + * postinstall notice or a warning can still reach the same stream, so the document is taken as the + * outermost braces rather than as the whole of stdout. + */ +export function parseReport(stdout: string): unknown { + const start = stdout.indexOf("{"); + const end = stdout.lastIndexOf("}"); + if (start < 0 || end <= start) return undefined; + try { return JSON.parse(stdout.slice(start, end + 1)); } catch { return undefined; } +} + +/** The tag the generated reporter prefixes its lines with, so the runner's own stderr is untouched. */ +const PROGRESS_TAG = "@@mobile-dev-step "; + +/** + * The generated config: the project's own, plus one reporter. + * + * Reporter *construction* must have no side effects — workers load the config too — so every write + * happens inside `onEvent`, which must also never throw and never block. A tagged line on stderr + * needs no socket, no temp output file and no cleanup beyond this config itself. + */ +export function progressConfigSource(configPath: string): string { + return [ + `// Generated by the mobile-dev Testing panel for one run, then deleted. Do not edit or commit.`, + `import base from ${JSON.stringify(pathToFileURL(configPath).href)};`, + ``, + `const resolved = typeof base === "function" ? await base() : base;`, + ``, + `export default {`, + ` ...resolved,`, + ` reporters: [`, + ` ...(resolved.reporters ?? []),`, + ` {`, + ` name: "mobile-dev-progress",`, + ` onEvent(event) {`, + ` if (event?.type !== "step") return;`, + ` const progress = event.progress ?? {};`, + ` if (progress.phase !== "start" && progress.phase !== "end") return;`, + ` try {`, + ` process.stderr.write(${JSON.stringify(PROGRESS_TAG)} + JSON.stringify({`, + ` stepIndex: progress.identity?.stepIndex,`, + ` phase: progress.phase,`, + ` api: progress.api,`, + ` label: progress.label,`, + ` status: progress.status,`, + ` error: progress.error?.message,`, + // The emitted file needs a literal backslash-n inside its own string, so this is escaped twice. + ` }) + "\\n");`, + ` } catch {`, + ` // A reporter that throws is quarantined for the rest of the run; progress is not`, + ` // worth losing the run's own output over.`, + ` }`, + ` },`, + ` },`, + ` ],`, + `};`, + ``, + ].join("\n"); +} + +/** Reads one tagged stderr line back. Anything else on that stream is the runner's own. */ +export function parseProgressLine(line: string): TestRunProgress | undefined { + if (!line.startsWith(PROGRESS_TAG)) return; + try { + const parsed = JSON.parse(line.slice(PROGRESS_TAG.length)) as Record<string, unknown>; + const stepIndex = typeof parsed.stepIndex === "number" ? parsed.stepIndex : undefined; + const phase = parsed.phase === "start" || parsed.phase === "end" ? parsed.phase : undefined; + if (stepIndex === undefined || phase === undefined) return; + return { + stepIndex, phase, api: str(parsed.api), label: str(parsed.label), + ...(str(parsed.status) ? { status: str(parsed.status) } : {}), + ...(str(parsed.error) ? { error: str(parsed.error).split("\n")[0]!.slice(0, 400) } : {}), + }; + } catch { return; } +} + +export interface TestRunRequest { + projectPath: string; + /** The test to run, relative to the project, e.g. `tests/a-recorded-flow.e2e.ts`. */ + relativePath: string; + /** The project's own `e2e.config.ts`, which the generated progress config spreads. */ + configPath?: string; + timeoutMs?: number; + /** + * A truncated copy of the test to run instead of the file itself — Play until here. + * + * The panel re-emits the flow from its first *n* steps, which is the saved file's own text with + * the later steps and nothing else removed, so every line it keeps is still at the line number + * it has in the file: the report's lines land on the preview's lines with no translation, and + * `EmittedTest.calls` indexes the same way because the truncated call list is a prefix of the + * full one. It is written beside the real test for one run and removed when the run ends — a + * sibling rather than a temp directory because the test is discovered through the project's own + * patterns (`tests/**`) and imports resolve from where it sits. + */ + untilSource?: string; + /** The e2e target to run against; without it the runner takes the config's first, whatever platform that is. */ + target?: string; +} + +/** + * Where the generated config goes: beside the project's own, at the project root. + * + * Not in `.e2e/`, tidy as that would be. **The runner takes the project root from the config + * file's own directory**, so a config one level down makes that directory the root and the run + * ends in `NO_TESTS: no test file matched "tests/**" under <project>/.e2e`. It is written for one + * run and removed when the run ends, so it is never a file anyone has to think about. + */ +const PROGRESS_CONFIG = "e2e.config.mobile-dev-progress.mts"; + +/** + * Where a Play-until-here run's truncated copy goes: beside the test it was cut from. + * + * The name is marked and the file is deleted when the run ends, so it is never a file anyone has + * to think about — and it keeps the `.e2e.ts` suffix because the project's own test patterns are + * what make a file discoverable at all; naming the selection does not exempt it from them. + */ +export function untilTestPath(relativePath: string): string { + return relativePath.endsWith(".e2e.ts") + ? `${relativePath.slice(0, -".e2e.ts".length)}.mobile-dev-until.e2e.ts` + : `${relativePath}.mobile-dev-until.e2e.ts`; +} + +type Spawner = typeof spawn; + +/** + * What a run needs doing before it may spawn anything. + * + * `release` gives the recorder's agent-device session back. The runner allows exactly one session + * per device, so a run started while the recorder holds it fails with `Device is already in use by + * session "mobile-dev-recorder"` — a refusal the user cannot connect to anything they did. It is a + * dependency of the run rather than something the caller is trusted to have done, because a caller + * who forgets produces exactly that failure. + */ +export interface TestRunDeps { + spawner?: Spawner; + release?: () => Promise<void>; + /** Called for each live `step` event, as it arrives. */ + onProgress?: (progress: TestRunProgress) => void; +} + +/** + * Spawns the project's own runner, not a global one: the test imports the project's + * `@e2e-dev/mobile`, and a different CLI is a different runner. `npx` is deliberately avoided — + * it will go to the network for a package that is not installed, which is a download the user did + * not ask for in the middle of a test run. + */ +export function runnerBin(projectPath: string): string { + const bin = join(projectPath, "node_modules/.bin/e2e"); + if (!existsSync(bin)) { + throw new ExpectedOperationError("invalid_input", + "This project has no e2e CLI installed, so the panel cannot run the test. Install the project's dependencies, then run it again."); + } + return bin; +} + +/** + * The environment the runner gets: the server's, minus `AGENT_DEVICE_STATE_DIR`. + * + * The server pins that variable so every daemon *this app* starts shares one state directory and + * they can take the device lease from one another (`pinStateDir` in `testing-device.ts`). Handing + * it to the project's own runner points the project's agent-device client at the app's state + * directory, and the two are not the same build: + * + * ``` + * ENGINE_FAILURE (launch): devices failed: Daemon (pid …, v0.21.33) is newer than this client + * (v0.21.20); refusing to replace it. + * ``` + * + * The runner refuses before it reaches a test, and the user sees a run that cannot start for a + * reason that has nothing to do with their test. Run exists to be the same run they would get from + * a terminal, so it gets the state directory a terminal would give it: its own. + */ +export function runnerEnv(environment: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + const { AGENT_DEVICE_STATE_DIR: _pinned, ...rest } = environment; + return { ...rest, FORCE_COLOR: "0", NO_COLOR: "1" }; +} + +/** + * Whether the run died over the generated config rather than over the test. + * + * Live progress is worth having and not worth losing a run over: if the runner will not take the + * generated config — an extension its loader does not accept, a project config that does not + * survive being spread — the run is retried exactly as it was before this feature existed. A + * progress spinner must never be the reason Run stopped working. + */ +function refusedTheConfig(result: TestRunResult): boolean { + return result.steps.length === 0 && /config/i.test(result.output ?? ""); +} + +export async function runRecordedTest(request: TestRunRequest, deps: TestRunDeps = {}): Promise<TestRunResult> { + const result = await runOnce(request, deps); + if (!refusedTheConfig(result) || !request.configPath || !deps.onProgress) return result; + // Without the progress sink nothing is generated, so this is the plain run. + const { onProgress: _dropped, ...plain } = deps; + const retried = await runOnce(request, plain); + // The retry is the answer only if it got further; otherwise the first failure is the real one. + return retried.steps.length > 0 || retried.status !== result.status ? retried : result; +} + +async function runOnce(request: TestRunRequest, deps: TestRunDeps): Promise<TestRunResult> { + const spawner = deps.spawner ?? spawn; + // Before the binary is even resolved: a project with no runner installed must not cost the user + // their recording session, and a release that happens after the spawn is no release at all. + const bin = runnerBin(request.projectPath); + await deps.release?.(); + // Live progress needs the generated config; without a config path to spread there is nothing to + // generate from, and the run still happens — it just lands all at once, from the report. + const generated = request.configPath && deps.onProgress + ? join(request.projectPath, PROGRESS_CONFIG) + : undefined; + if (generated && request.configPath) await writeFile(generated, progressConfigSource(request.configPath)); + // Play until here runs a truncated copy of the file, not the file: see `untilSource`. The real + // test stays exactly as the user saved it, and the copy goes when the run does. + const runPath = request.untilSource === undefined ? request.relativePath : untilTestPath(request.relativePath); + const partial = request.untilSource === undefined ? undefined : join(request.projectPath, runPath); + if (partial && request.untilSource !== undefined) await writeFile(partial, request.untilSource); + const discard = () => { + if (generated) void rm(generated, { force: true }); + if (partial) void rm(partial, { force: true }); + }; + const child = spawner(bin, [ + "run", runPath, "--reporter", "json", + ...(request.target ? ["--target", request.target] : []), + ...(generated ? ["--config", PROGRESS_CONFIG] : []), + ], { + cwd: request.projectPath, + stdio: ["ignore", "pipe", "pipe"], + env: runnerEnv(), + }); + let stdout = "", stderr = ""; + child.stdout?.setEncoding("utf8"); + child.stderr?.setEncoding("utf8"); + child.stdout?.on("data", chunk => { stdout += chunk; }); + // Progress arrives a line at a time, interleaved with the runner's own stderr, so the stream is + // split on newlines and a partial tail is held over for the next chunk. + let pending = ""; + child.stderr?.on("data", chunk => { + stderr += chunk; + if (!deps.onProgress) return; + pending += chunk; + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) { + const progress = parseProgressLine(line); + if (progress) deps.onProgress(progress); + } + }); + const timeoutMs = request.timeoutMs ?? 10 * 60_000; + const exitCode = await new Promise<number>((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGTERM"); + discard(); + reject(new ExpectedOperationError("resource_limit", `The test run did not finish within ${Math.round(timeoutMs / 1000)}s.`)); + }, timeoutMs); + timer.unref?.(); + child.on("error", error => { clearTimeout(timer); discard(); reject(error); }); + child.on("close", code => { clearTimeout(timer); resolve(code ?? 1); }); + }); + if (generated) await rm(generated, { force: true }); + if (partial) await rm(partial, { force: true }); + const report = parseReport(stdout); + // The report attributes its steps to the file the runner actually ran, which for a partial run + // is the copy; its line numbers are the real file's, so the panel needs no translation. + const steps = report ? reportSteps(report, runPath) : []; + const durationMs = report ? reportDuration(report) : undefined; + return { + status: reportStatus(report, exitCode), + exitCode, + steps, + ...(durationMs === undefined ? {} : { durationMs }), + // Only when there is nothing to show on the lines: an exit with no report is the runner + // refusing before it got to the test, and its own words are the only explanation there is. + ...(steps.length === 0 ? { output: tail(`${stdout}\n${stderr}`) } : {}), + }; +} diff --git a/packages/core/src/testing-tools.ts b/packages/core/src/testing-tools.ts new file mode 100644 index 00000000..eeca1299 --- /dev/null +++ b/packages/core/src/testing-tools.ts @@ -0,0 +1,309 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; +import { z } from "zod"; +import { errorMessage } from "@mobile-dev/shared/protocol.ts"; +import { captureServerError } from "./telemetry.ts"; +import type { TestingProjectChoices } from "./testing-projects.ts"; +import { deleteTestFile, discardStagedTestFile, inspectProject, listTestFiles, previewTestFile, resolveTestRunPaths, saveStagedTestFile, stageTestFile, writeTestFile } from "./testing-projects.ts"; +import { TestDraftStore } from "./testing-drafts.ts"; +import { MAX_DRAFT_STEPS, recordedStepSchema, testDraftIdSchema } from "@mobile-dev/shared/test-drafts.ts"; +import type { TestDraft } from "@mobile-dev/shared/test-drafts.ts"; +import { TestingDevices } from "./testing-device.ts"; +import { runRecordedTest } from "./testing-run.ts"; +import type { TestRunProgress } from "./testing-run.ts"; +import { deviceIdSchema } from "@mobile-dev/shared/panel-target.ts"; + +const read = { readOnlyHint: true, destructiveHint: false, openWorldHint: false }; +const write = { readOnlyHint: false, destructiveHint: false, openWorldHint: false }; +const destructive = { readOnlyHint: false, destructiveHint: true, openWorldHint: false }; +/** The Testing panel drives these itself; the model has its own e2e MCP server for test work. */ +const appVisibility = { ui: { visibility: ["app"] as ("app" | "model")[] } }; + +const projectPath = z.string().min(1).max(4096); + +function safe<T>(handler: (input: T) => Promise<CallToolResult>) { + return async (input: T): Promise<CallToolResult> => { + try { + return await handler(input); + } catch (error) { + captureServerError(error, "testing.tool"); + return { isError: true, content: [{ type: "text", text: errorMessage(error) }] }; + } + }; +} + +function dataResult(data: Record<string, unknown>, message?: string): CallToolResult { + return { content: [{ type: "text", text: message ?? JSON.stringify(data) }], structuredContent: data }; +} + +/** The Testing panel's project selection, and the device tree its recorder names selectors from. */ +/** Returns the shutdown the plugin has to run: the recorder's device session, handed back. */ +export function registerTestingTools(server: McpServer, projects: TestingProjectChoices, devices = new TestingDevices(), drafts = new TestDraftStore()): () => Promise<void> { + /** + * The live progress of the run in flight. + * + * A tool call is one request and one response, so the run's own call cannot report as it goes. + * The panel polls this one instead, which is a few hundred bytes a second against a local server + * for as long as a run takes — cheaper than holding a notification channel open for a feature + * whose whole output is a spinner moving down a dozen lines. + */ + let progress: TestRunProgress[] = []; + let running = false; + registerAppTool(server, "mobile_testing_projects", { + title: "List testing projects", + description: "List the e2e project folders the Testing panel can record against, most recent first.", + inputSchema: {}, + annotations: read, + _meta: appVisibility, + }, safe(async () => dataResult({ projects: await projects.list() }))); + + registerAppTool(server, "mobile_testing_choose_project", { + title: "Choose a testing project", + description: + "Choose the project the Testing panel records against: pass projectPath to select one the panel already lists, or nothing to open the system folder picker. The folder must hold an e2e.config.ts. The choice persists across sessions.", + inputSchema: { projectPath: projectPath.optional() }, + annotations: write, + _meta: appVisibility, + }, safe(async ({ projectPath: selected }: { projectPath?: string }) => { + const project = selected === undefined ? await projects.choose() : await projects.record(selected); + return dataResult(project ? { project } : {}, project ? undefined : "No project was chosen."); + })); + + registerAppTool(server, "mobile_testing_write_file", { + title: "Save a recorded test", + description: + "Write a recorded test into the project's tests/ folder. Without overwrite an existing file is reported back untouched instead of being replaced.", + inputSchema: { + projectPath, + relativePath: z.string().min(1).max(512).describe("Path under the project, inside tests/, ending in .e2e.ts."), + contents: z.string().max(256 * 1024), + overwrite: z.boolean().optional(), + }, + // Replacing someone's test file is destructive, so the host asks before this runs. + annotations: destructive, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath, contents, overwrite }: { projectPath: string; relativePath: string; contents: string; overwrite?: boolean }) => { + const result = await writeTestFile(selected, relativePath, contents, overwrite ?? false); + return dataResult({ ...result }, result.written ? `Saved ${result.path}` : `${result.path} already exists.`); + })); + + /** + * The tests the project already holds. + * + * The other half of the panel's list. A recording that has been written into the project stops + * being a draft, and the draft is deleted at that point — so until this existed, pressing Save + * made the test leave the list altogether. Read from the folder rather than remembered, so a + * test written by hand or by an agent is listed beside the recorded ones. + */ + registerAppTool(server, "mobile_testing_tests", { + title: "List the project's tests", + description: "List the .e2e.ts files already in the project's tests/ folder, newest first.", + inputSchema: { projectPath }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ projectPath: selected }: { projectPath: string }) => + dataResult({ tests: await listTestFiles(selected) }))); + + registerAppTool(server, "mobile_testing_read_file", { + title: "Preview a project test", + description: "Read the source of one .e2e.ts file in the project's tests/ folder for the Testing panel preview.", + inputSchema: { projectPath, relativePath: z.string().min(1).max(512) }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath }: { projectPath: string; relativePath: string }) => + dataResult(await previewTestFile(selected, relativePath)))); + + registerAppTool(server, "mobile_testing_stage_file", { + title: "Keep project test edits temporarily", + description: "Keep edits to a project test in a temporary working copy without changing the project file.", + inputSchema: { projectPath, relativePath: z.string().min(1).max(512), originalHash: z.string().length(64), source: z.string().max(256 * 1024) }, + annotations: write, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath, originalHash, source }: { projectPath: string; relativePath: string; originalHash: string; source: string }) => { + await stageTestFile(selected, relativePath, originalHash, source); + return dataResult({ staged: true }); + })); + + registerAppTool(server, "mobile_testing_save_staged_file", { + title: "Save project test edits", + description: "Write a staged test to its original project file if that file has not changed since editing began.", + inputSchema: { projectPath, relativePath: z.string().min(1).max(512) }, + annotations: destructive, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath }: { projectPath: string; relativePath: string }) => + dataResult(await saveStagedTestFile(selected, relativePath)))); + + registerAppTool(server, "mobile_testing_discard_staged_file", { + title: "Undo project test edits", + description: "Discard a temporary working copy and return the unchanged project test.", + inputSchema: { projectPath, relativePath: z.string().min(1).max(512) }, + annotations: write, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath }: { projectPath: string; relativePath: string }) => + dataResult(await discardStagedTestFile(selected, relativePath)))); + + registerAppTool(server, "mobile_testing_delete_file", { + title: "Delete one of the project's tests", + description: "Delete one .e2e.ts file from the project's tests/ folder. The panel asks before calling this; the file is not recoverable from here.", + inputSchema: { + projectPath, + relativePath: z.string().min(1).max(512).describe("The test to delete, relative to the project, e.g. tests/a-recorded-flow.e2e.ts."), + }, + // Removing a file from the user's repository is exactly what the host should ask about. + annotations: destructive, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath }: { projectPath: string; relativePath: string }) => { + const result = await deleteTestFile(selected, relativePath); + return dataResult({ ...result }, result.removed ? `Deleted ${result.path}` : `${result.path} was already gone.`); + })); + + /** + * The unsaved recordings. + * + * A flow lives in the panel document until Save writes it into the project, and the host rebuilds + * that document whenever the user looks at another tool or another chat — so a flow they have not + * saved, or one their agent recorded while they were reading the chat, used to be gone when they + * came back. The panel autosaves into these instead, and lists them so a flow can be reopened, + * swapped for another or thrown away. Nothing here touches the user's project. + */ + registerAppTool(server, "mobile_testing_drafts", { + title: "List unsaved recordings", + description: "List the recorded tests that have not been written into a project yet, newest first. Pass projectPath to list one project's drafts.", + inputSchema: { projectPath: projectPath.optional() }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ projectPath: selected }: { projectPath?: string }) => + dataResult({ drafts: await drafts.list(selected ?? "") }))); + + registerAppTool(server, "mobile_testing_draft_read", { + title: "Read an unsaved recording", + description: "Read one unsaved recording, with its steps, so the panel can put it back in the step list.", + inputSchema: { id: testDraftIdSchema }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ id }: { id: string }) => { + const draft = await drafts.read(id); + return dataResult(draft ? { draft } : {}, draft ? undefined : "That recording is no longer there."); + })); + + registerAppTool(server, "mobile_testing_draft_save", { + title: "Keep an unsaved recording", + description: + "Write the recording in the panel to the plugin's own storage so it survives the panel being rebuilt. Nothing is written into the user's project; that is mobile_testing_write_file.", + inputSchema: { + id: testDraftIdSchema.optional().describe("The draft to replace. Omit to start a new one."), + projectPath, + title: z.string().max(160), + fileName: z.string().max(256).optional(), + origin: z.enum(["user", "agent"]).optional(), + steps: z.array(recordedStepSchema).max(MAX_DRAFT_STEPS), + source: z.string().max(256 * 1024).optional(), + }, + // The plugin's own state, replaced wholesale by the panel that owns it. + annotations: write, + _meta: appVisibility, + }, safe(async (input: { id?: string; projectPath: string; title: string; fileName?: string; origin?: "user" | "agent"; steps: TestDraft["steps"]; source?: string }) => { + const draft = await drafts.save({ ...input, steps: input.steps }); + return dataResult({ id: draft.id, updatedAt: draft.updatedAt }); + })); + + registerAppTool(server, "mobile_testing_draft_delete", { + title: "Delete an unsaved recording", + description: "Forget one unsaved recording. The steps are gone; a recording already written into the project is untouched.", + inputSchema: { id: testDraftIdSchema }, + // Throwing away a flow the user recorded and never saved is not recoverable. + annotations: destructive, + _meta: appVisibility, + }, safe(async ({ id }: { id: string }) => { + const removed = await drafts.remove(id); + return dataResult({ removed }, removed ? "Deleted the recording." : "That recording was already gone."); + })); + + registerAppTool(server, "mobile_testing_project_config", { + title: "Read a testing project's config", + description: "Read the targets of a project's e2e config: the target names, their platforms, and the bundle ids to launch.", + inputSchema: { projectPath }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ projectPath: selected }: { projectPath: string }) => + dataResult({ ...await inspectProject(selected) }))); + + registerAppTool(server, "mobile_testing_read_tree", { + title: "Read the runner's element tree", + description: + "Read the element tree the e2e runner queries, for the app the device is already showing. The recorder names selectors from this so role vocabulary and name sources match what the test will run against.", + inputSchema: { + deviceId: deviceIdSchema, + platform: z.enum(["ios", "android"]).describe("The mirrored device's platform: an iOS simulator's deviceId is its UDID, an Android emulator's its adb serial."), + projectPath, + bundleId: z.string().min(1).max(512).describe("The target's bundle id, from the project's e2e config."), + }, + annotations: read, + _meta: appVisibility, + }, safe(async ({ deviceId, platform, projectPath: selected, bundleId }: { deviceId: string; platform: "ios" | "android"; projectPath: string; bundleId: string }) => { + // The refs go no further than the server: the panel records selectors and drives the mirror, + // so an agent-device handle per element would be a few kilobytes a read that nothing reads. + const { tree, screen } = await devices.read({ deviceId, platform, projectPath: selected, bundleId }); + return dataResult({ tree, ...(screen ? { screen } : {}) }); + })); + + registerAppTool(server, "mobile_testing_run_test", { + title: "Run a recorded test", + description: + "Run one saved test with the project's own e2e CLI and report each step with the line it came from. The recorder's device session is released first: the runner allows one session per device, so a run started while the recorder holds it fails with \"Device is already in use\".", + inputSchema: { + projectPath, + relativePath: z.string().min(1).max(512).describe("The test to run, relative to the project, e.g. tests/a-recorded-flow.e2e.ts."), + configPath: z.string().min(1).max(4096).optional().describe("The project's e2e config, so the run can report live per-step progress."), + timeoutMs: z.number().int().min(10_000).max(30 * 60_000).optional(), + target: z.string().min(1).max(256).optional().describe("The e2e target name to run against, so a flow recorded on Android does not run on the config's first (iOS) target."), + untilSource: z.string().min(1).max(400_000).optional().describe( + "Run this truncated copy of the test instead of the file itself, for \"play until here\": the same flow cut off after one step. It is written beside the test for the one run and removed afterwards; the file itself is not touched."), + }, + // A test run drives the app on the device: it taps, types and changes the app's own state. + annotations: destructive, + _meta: appVisibility, + }, safe(async ({ projectPath: selected, relativePath, configPath, timeoutMs, target, untilSource }: { projectPath: string; relativePath: string; configPath?: string; timeoutMs?: number; target?: string; untilSource?: string }) => { + // The release is a dependency of the run, not a step the caller is trusted to have taken: see + // `TestRunDeps`. The panel releases too, but a caller who forgets must not break the run. + progress = []; + running = true; + try { + const paths = await resolveTestRunPaths(selected, relativePath, configPath); + const result = await runRecordedTest( + { ...paths, ...(timeoutMs === undefined ? {} : { timeoutMs }), ...(target ? { target } : {}), + ...(untilSource === undefined ? {} : { untilSource }) }, + { release: () => devices.release(), onProgress: entry => { progress.push(entry); } }); + return dataResult({ ...result }, `${relativePath} ${result.status}.`); + } finally { + running = false; + } + })); + + registerAppTool(server, "mobile_testing_run_progress", { + title: "Read a run's progress", + description: + "Read the steps the run in flight has started and finished so far, as the runner's own event spine reports them. Each entry carries the step's index within its test, which is its position among the file's awaited calls.", + inputSchema: {}, + annotations: read, + _meta: appVisibility, + }, safe(async () => dataResult({ running, progress: [...progress] }))); + + registerAppTool(server, "mobile_testing_release_device", { + title: "Release the recorder's device", + description: + "Close the recorder's session on the device. A session left open makes the next `e2e run` fail because the device is already in use, so the panel calls this whenever it stops recording.", + inputSchema: {}, + annotations: write, + _meta: appVisibility, + }, safe(async () => { await devices.release(); return dataResult({ released: true }, "Released the device."); })); + + /** + * A server that goes away without releasing leaves the claim behind, and the daemon that holds it + * outlives the server. The user then sees their own `e2e run` refuse to boot the device — "has an + * ownership claim that could not be verified" — with nothing on screen connecting it to a panel + * they closed. Every ordinary path already releases; this is the one that is not ordinary. + */ + return () => devices.release(); +} diff --git a/packages/core/test/jsdom-env.ts b/packages/core/test/jsdom-env.ts index c1e93473..04d8fbf8 100644 --- a/packages/core/test/jsdom-env.ts +++ b/packages/core/test/jsdom-env.ts @@ -36,6 +36,19 @@ export async function renderEnvironment<T = Record<string, unknown>>( const w = dom.window; const ResizeObserver = class { observe() {} unobserve() {} disconnect() {} }; const DOMMatrixReadOnly = class { m22 = 1; constructor() {} }; + // JSDOM has no `DOMRect`, and a layout-measuring component constructs one. Measurements in this + // environment are all zero anyway: the point of these tests is the DOM that is rendered. + const DOMRectStub = w.DOMRect ?? class { + x: number; y: number; width: number; height: number; + constructor(x = 0, y = 0, width = 0, height = 0) { + this.x = x; this.y = y; this.width = width; this.height = height; + } + get top() { return this.y; } + get left() { return this.x; } + get right() { return this.x + this.width; } + get bottom() { return this.y + this.height; } + toJSON() { return { ...this }; } + }; const globals: Record<string, unknown> = { window: w, document: w.document, navigator: w.navigator, HTMLElement: w.HTMLElement, HTMLFormElement: w.HTMLFormElement, HTMLButtonElement: w.HTMLButtonElement, @@ -44,7 +57,7 @@ export async function renderEnvironment<T = Record<string, unknown>>( Element: w.Element, DocumentFragment: w.DocumentFragment, MutationObserver: w.MutationObserver, localStorage: w.localStorage, getComputedStyle: w.getComputedStyle.bind(w), requestAnimationFrame: w.requestAnimationFrame.bind(w), cancelAnimationFrame: w.cancelAnimationFrame.bind(w), - ResizeObserver, IS_REACT_ACT_ENVIRONMENT: true, + ResizeObserver, DOMRect: DOMRectStub, IS_REACT_ACT_ENVIRONMENT: true, ...(options.graph ? { DOMMatrixReadOnly } : {}), ...options.extraGlobals, }; @@ -54,8 +67,11 @@ export async function renderEnvironment<T = Record<string, unknown>>( previous.set(key, Object.getOwnPropertyDescriptor(globalThis, key)); Object.defineProperty(globalThis, key, { value, configurable: true, writable: true }); } + // On the window as well as on the global: a component that reads `window.ResizeObserver` — the + // resizable panel group does — finds nothing on a JSDOM window, which has none of its own. + Object.defineProperty(w, "ResizeObserver", { value: ResizeObserver, configurable: true }); + Object.defineProperty(w, "DOMRect", { value: DOMRectStub, configurable: true }); if (options.graph) { - Object.defineProperty(w, "ResizeObserver", { value: ResizeObserver, configurable: true }); Object.defineProperty(w, "DOMMatrixReadOnly", { value: DOMMatrixReadOnly, configurable: true }); } @@ -76,7 +92,10 @@ export async function renderEnvironment<T = Record<string, unknown>>( await build({ stdin: { contents: options.exports, resolveDir: process.cwd(), loader: "ts" }, outfile, bundle: true, format: "esm", platform: "node", jsx: "automatic", - external: ["react", "react/*", "react-dom", "react-dom/*", "radix-ui", "lucide-react", "@base-ui/react/*", "@xyflow/react"], + // `react-resizable-panels` ships a CJS build whose `require("react/jsx-runtime")` cannot be + // resolved once React itself is external to an ESM bundle. Left external, Node loads the + // package's own ESM entry and the require never happens. + external: ["react", "react/*", "react-dom", "react-dom/*", "radix-ui", "lucide-react", "@base-ui/react/*", "@xyflow/react", "react-resizable-panels"], }); return { module: (await import(pathToFileURL(outfile).href)) as T, diff --git a/packages/core/test/mobile-dev-skill.test.ts b/packages/core/test/mobile-dev-skill.test.ts index be03fff1..0f221c39 100644 --- a/packages/core/test/mobile-dev-skill.test.ts +++ b/packages/core/test/mobile-dev-skill.test.ts @@ -11,3 +11,50 @@ test("App Flow opens saved and runtime-only screens without extra device verific assert.match(skill, /When either replay tool returns `ok: true`, stop/); assert.match(skill, /Do not take a snapshot, verify the destination, open an agent-device session/); }); + +test("the testing skill records, hands over, and never saves or runs", async () => { + const skill = await readFile(new URL("../../skills/mobile-dev-testing/SKILL.md", import.meta.url), "utf8"); + + assert.match(skill, /iOS or Android app/); + assert.match(skill, /booted iOS simulator or Android emulator in the mirror/); + + // The loop, in the order the tools enforce it. + assert.match(skill, /mobile_testing_agent_begin.*mobile_testing_agent_screen.*mobile_testing_agent_step.*mobile_testing_agent_finish/s); + assert.match(skill, /Read the screen again after every step that acts/); + // The prerequisites are the user's to set: an agent that opens the folder picker opens a system + // dialog inside its own turn. + assert.match(skill, /\*\*Never choose the project for them\*\*/); + // Pinned locators are the point of recording; a sentence is the fallback for an element the + // screen read reported no selector for. + assert.match(skill, /## Never write a locator/); + assert.match(skill, /no `selector` at all/); + assert.match(skill, /\*\*Never save\.\*\*/); + assert.match(skill, /\*\*Never run the test\.\*\*/); + // And the handover, which is the whole deliverable: the user adjusts and saves. + assert.match(skill, /in the Testing panel, unsaved/); +}); + +test("the testing skill says where a flow ends, not only where it starts", async () => { + const skill = await readFile(new URL("../../skills/mobile-dev-testing/SKILL.md", import.meta.url), "utf8"); + + // The bug this rule exists for: told to record "the Explore screen", the agent read the name as + // a starting point, navigated on, and recorded the next screen's heading, sections and search. + assert.match(skill, /## Where the flow ends/); + assert.match(skill, /\*\*The screen the user named is the scope of the recording, not the place it starts\.\*\*/); + assert.match(skill, /assert the arrival with \*\*one\*\* assertion/); + assert.match(skill, /Record nothing on\s+the screen you arrived at/); + // The hint the tool gives it, and the one case where crossing screens is the point. + assert.match(skill, /`navigated: true`/); + assert.match(skill, /\*\*A flow name is not a boundary\.\*\*/); + // Saying what was left out costs a sentence; recording it costs the user an edit. + assert.match(skill, /I stopped at Explore List/); + // A disclosure left open is a screen the next step resolves against and the test will not reach. + assert.match(skill, /A control you \*\*open\*\* is part of the screen, so assert what it revealed/); +}); + +test("the testing skill ships from every plugin that has the panel", async () => { + const { sharedSkills } = await import("../../../tooling/plugin-hosts.mjs"); + assert.ok(sharedSkills.includes("packages/skills/mobile-dev-testing"), "Codex and Cursor"); + const claude = JSON.parse(await readFile(new URL("../../../apps/claude/plugin-host.json", import.meta.url), "utf8")); + assert.ok(claude.skills.includes("../../packages/skills/mobile-dev-testing"), "Claude Code"); +}); diff --git a/packages/core/test/plugin.test.ts b/packages/core/test/plugin.test.ts index 0d1ed246..0b0c999e 100644 --- a/packages/core/test/plugin.test.ts +++ b/packages/core/test/plugin.test.ts @@ -497,6 +497,40 @@ test("a stale accessibility bridge restarts once before reading the UI again", a assert.equal(restarts.length, 1); }); +test("a stale bridge that answers ok is recovered too, and a healthy read is never retried", async t => { + let now = 0; + t.mock.method(Date, "now", () => now); + let responses: unknown[] = []; + const reads: number[] = []; + const server = createServer((_request, response) => { reads.push(1); response.end(JSON.stringify(responses.shift() ?? { ok: true, tree: { label: "HorseTinder" } })); }); + await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve)); + t.after(() => server.close()); + const restarts: string[] = []; + const baguette = new Baguette(`http://127.0.0.1:${(server.address() as AddressInfo).port}`, async udid => { restarts.push(udid); }); + t.after(() => baguette.dispose()); + + // The bridge goes stale without failing: it answers with the simulator's chrome under a root + // that has no app name. Downstream this looks like an app with nothing accessible in it. + responses = [{ ok: true, tree: { label: null } }, { ok: true, tree: { label: "HorseTinder" } }]; + assert.equal((await baguette.describeUi(UDID)).tree?.label, "HorseTinder"); + assert.deepEqual(restarts, [UDID]); + + // A healthy read costs one request and never restarts anything. + now = 20000; + reads.length = 0; + responses = [{ ok: true, tree: { label: "HorseTinder" } }]; + assert.equal((await baguette.describeUi(UDID)).tree?.label, "HorseTinder"); + assert.equal(reads.length, 1); + assert.equal(restarts.length, 1); + + // Still unnamed after a restart, or too soon to restart again: the caller gets what there is + // rather than an error, so the panel can say what it sees. + now = 40000; + responses = [{ ok: true, tree: { label: null } }, { ok: true, tree: { label: null } }]; + assert.equal((await baguette.describeUi(UDID)).tree?.label, null); + assert.equal(restarts.length, 2); +}); + test("panel input reaches the native socket during a pending refresh and blocks after it reports Device Hub", { timeout: 2000 }, async t => { let now = 0; t.mock.method(Date, "now", () => now); diff --git a/packages/core/test/react-native-inspector.test.ts b/packages/core/test/react-native-inspector.test.ts index 82908c59..786a7bbb 100644 --- a/packages/core/test/react-native-inspector.test.ts +++ b/packages/core/test/react-native-inspector.test.ts @@ -407,3 +407,38 @@ test("the collector separates selected creation stacks from owner stacks and kee assert.deepEqual(exact.tree[0].creationStackIds, [0]); assert.deepEqual(exact.tree[0].ownerStackIds, [1]); }); + +test("a text node holding nested text is named by the whole string, not by its component", () => { + // The Explore screen's copy: a <Text> whose children are strings interleaved with nested <Text>. + // Reading `children` directly yields nothing, so the node used to be labelled "RCTText" — and + // iOS accessibility does not cover React Native views, so nothing else would ever name it. + const element = (type: string, children: unknown) => ({ type, props: { children } }); + const paragraph = nativeFiber("RCTText", { x: 32, y: 300, width: 327, height: 60 }, { + children: [ + "This app has two screens:\n ", + element("ThemedText", "src/app/index.tsx"), + " and ", + element("ThemedText", ["src/app/", element("ThemedText", "explore.tsx")]), + ".", + ], + }); + const result = collectFibers({ ...nativeFiber("RCTView", { x: 0, y: 0, width: 400, height: 900 }), child: paragraph }); + const [, text] = screenComponents(JSON.parse(JSON.stringify(result.tree))); + assert.equal(text.label, "This app has two screens: src/app/index.tsx and src/app/explore.tsx."); +}); + +test("a container is never labelled with the copy inside it", () => { + // Only a text component flattens nested text. A container's accessible name is accessibility's + // business — inventing one here would name a Pressable after the words it happens to wrap, and + // a recorded selector built on that name is one the runner cannot resolve. + const view = nativeFiber("RCTView", { x: 0, y: 0, width: 400, height: 900 }, { + children: [{ type: "RCTText", props: { children: "Heading" } }, { type: "Card", props: { children: "Body copy" } }], + }); + const [root] = screenComponents(JSON.parse(JSON.stringify(collectFibers(view).tree))); + assert.equal(root.label, "RCTView", "the container keeps no label of its own"); + + // A text component with a single string child still reads as it always did. + const plain = nativeFiber("RCTText", { x: 0, y: 0, width: 100, height: 20 }, { children: "Heading" }); + const [text] = screenComponents(JSON.parse(JSON.stringify(collectFibers(plain).tree))); + assert.equal(text.label, "Heading"); +}); diff --git a/packages/core/test/testing-agent-panel.test.ts b/packages/core/test/testing-agent-panel.test.ts new file mode 100644 index 00000000..7c55c1a1 --- /dev/null +++ b/packages/core/test/testing-agent-panel.test.ts @@ -0,0 +1,169 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { renderEnvironment } from "./jsdom-env.ts"; + +const DEVICE = { udid: "udid", name: "iPhone 17 Pro", state: "Booted", platform: "ios", kind: "simulator" }; +// `useSyncExternalStore` compares snapshots by identity, so this is a value, not a fresh object. +const DEVICE_SNAPSHOT = { device: DEVICE }; + +/** One step as the server's agent recording reports it: the panel's own `RecordedStep`, over MCP. */ +function step(id: string, label: string) { + return { id, kind: "tap", label, selector: { kind: "text", text: label } }; +} + +/** + * The Testing panel over a host whose tool calls are answered from a script. + * + * `agent` is the state the server would report, which the test moves between polls; everything + * else answers with the empty shape the panel copes with, so the test is about the agent recording + * and not about project loading. + */ +async function panelEnvironment(t: Parameters<typeof renderEnvironment>[0]) { + const { module, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingPanel } from "./packages/ui/src/testing-panel.ts";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";\n' + + 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";', + slug: "testing-agent-panel", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const agent: Record<string, unknown> = { active: false, status: "idle", revision: 0, title: "", goal: "", steps: [], note: "", error: "" }; + const host = { + callServerTool: async (request: { name: string; arguments: Record<string, unknown> }) => { + calls.push(request); + if (request.name === "mobile_testing_agent_state") return { content: [], structuredContent: { ...agent } }; + if (request.name === "mobile_testing_projects") return { content: [], structuredContent: { projects: [] } }; + return { content: [], structuredContent: {} }; + }, + }; + const listeners = new Set<() => void>(); + let snapshot = DEVICE_SNAPSHOT; + const deviceApps = { + subscribe: (listener: () => void) => { listeners.add(listener); return () => { listeners.delete(listener); }; }, + getSnapshot: () => snapshot, + }; + const panel = new module.TestingPanel(host, deviceApps); + // The harness keeps one teardown, so they are chained here: a panel left undisposed keeps its + // poll timer, and the test run never exits. + const extra: (() => Promise<void> | void)[] = []; + onTeardown(async () => { for (const fn of extra.reverse()) await fn(); panel.dispose(); }); + const tick = async () => { await new Promise(resolve => setTimeout(resolve, 0)); await new Promise(resolve => setTimeout(resolve, 0)); }; + return { ...module, dom, onTeardown: (fn: () => Promise<void> | void) => { extra.push(fn); }, panel, calls, agent, tick, + setDevice: (next: unknown) => { snapshot = { device: next } as never; for (const listener of listeners) listener(); } }; +} + +test("the panel reports its selection, and only when it changes", async t => { + const { panel, calls, tick } = await panelEnvironment(t); + panel.setAvailable(true); + await tick(); + const context = () => calls.filter(entry => entry.name === "mobile_testing_set_context"); + // Nothing is reported as a selection until all three halves are there: an agent recording + // against half a selection is the refusal it would get anyway, one round trip later. + assert.deepEqual(context().at(-1)?.arguments, { projectPath: "", deviceId: DEVICE.udid, platform: "ios", bundleId: "" }); + const before = context().length; + panel.setFileName("whatever"); + await tick(); + assert.equal(context().length, before, "an unchanged selection is not reported again"); +}); + +test("a session the agent starts opens the Testing tab and adopts its steps live", async t => { + const { panel, agent, tick, testRecorder } = await panelEnvironment(t); + let opened = 0; + panel.onAgentRecordingStarted(() => { opened += 1; }); + panel.setAvailable(true); + await tick(); + assert.equal(opened, 0, "an idle server opens nothing"); + + Object.assign(agent, { active: true, status: "recording", revision: 1, title: "explore details", goal: "cover the landmarks", steps: [step("a", 'Tap "Explore"')] }); + await (panel as any).readAgentState(); + assert.equal(opened, 1, "the user asked their agent for a test; the steps have to be somewhere they can see them"); + assert.equal(panel.getSnapshot().agent.active, true); + // The same step list, the same preview, the same emitter — deliberately not a second state + // machine, which is what makes Save and the step editor work unchanged. + assert.deepEqual(testRecorder.getSnapshot().steps.map((entry: any) => entry.id), ["a"]); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, DEVICE.udid); + assert.equal(testRecorder.isAgentRecording(DEVICE.udid), true); + assert.match(testRecorder.source, /await screen\.getByText\("Tap \\"Explore\\""\)\.tap\(\);/); + + Object.assign(agent, { revision: 2, steps: [step("a", 'Tap "Explore"'), step("b", 'Tap "Paris"')] }); + await (panel as any).readAgentState(); + assert.deepEqual(testRecorder.getSnapshot().steps.map((entry: any) => entry.id), ["a", "b"]); + assert.equal(opened, 1, "the tab is opened once per session, not once per step"); +}); + +test("a finished session hands the flow back, dirty and unsaved", async t => { + const { panel, agent, tick, testRecorder } = await panelEnvironment(t); + panel.setAvailable(true); + await tick(); + Object.assign(agent, { active: true, status: "recording", revision: 1, title: "t", goal: "g", steps: [step("a", "Tap")] }); + await (panel as any).readAgentState(); + Object.assign(agent, { active: false, status: "finished", revision: 2, note: "the Explore Details landmarks" }); + await (panel as any).readAgentState(); + + // The banner clears and the mirror is the user's again, with the steps still there to adjust. + assert.equal(testRecorder.isAgentRecording(DEVICE.udid), false); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, "", "a flow left armed turns the next stray tap into a step the agent never recorded"); + assert.equal(testRecorder.getSnapshot().steps.length, 1); + assert.equal(panel.getSnapshot().agent.status, "finished"); + assert.equal(panel.dirty, true, "nothing was saved, so everything is unsaved"); + assert.equal(panel.canRun, false, "no project is selected in this fixture"); +}); + +test("Stop drops the session at the server and hands the mirror back at once", async t => { + const { panel, agent, calls, tick, testRecorder } = await panelEnvironment(t); + panel.setAvailable(true); + await tick(); + Object.assign(agent, { active: true, status: "recording", revision: 1, title: "t", goal: "g", steps: [step("a", "Tap")] }); + await (panel as any).readAgentState(); + + await panel.stopAgent(); + assert.equal(calls.some(entry => entry.name === "mobile_testing_agent_abort"), true); + // Not on the next tick: the user pressed Stop to get the mirror back. + assert.equal(testRecorder.isAgentRecording(DEVICE.udid), false); + assert.equal(panel.getSnapshot().agent.status, "aborted"); + assert.equal(testRecorder.getSnapshot().steps.length, 1, "whatever was recorded is the user's to keep"); +}); + +test("the user cannot start a second recording over the agent's", async t => { + const { panel, agent, tick, testRecorder } = await panelEnvironment(t); + panel.setAvailable(true); + await tick(); + Object.assign(agent, { active: true, status: "recording", revision: 1, title: "t", goal: "g", steps: [] }); + await (panel as any).readAgentState(); + testRecorder.start(DEVICE.udid, "ios", false); + assert.equal(testRecorder.isAgentRecording(DEVICE.udid), true, "Record is refused while the agent drives the device"); +}); + +test("the banner says who is driving, and its Stop is the one beside the goal", async t => { + const { panel, agent, dom, onTeardown, TestingView, tick } = await panelEnvironment(t); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + panel.setAvailable(true); + await tick(); + const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT }; + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps })); }); + onTeardown(async () => { await act(async () => { root.unmount(); }); }); + + Object.assign(agent, { active: true, status: "recording", revision: 1, title: "t", goal: "cover the Explore Details landmarks", steps: [step("a", "Tap")] }); + await act(async () => { await (panel as any).readAgentState(); }); + const banner = () => dom.window.document.querySelector("[data-testing-agent-banner]"); + assert.match(banner()?.textContent ?? "", /Agent is recording — cover the Explore Details landmarks/); + assert.match(banner()?.textContent ?? "", /1 step/); + assert.ok(dom.window.document.querySelector("[data-testing-agent-stop]"), "the user must be able to take the mirror back without going via their agent"); + // One Stop, beside what it stops: the recorder's own Stop is not drawn while the agent drives. + assert.equal(dom.window.document.querySelector("[data-testing-stop]"), null); + assert.equal((dom.window.document.querySelector("[data-testing-run]") as HTMLButtonElement).disabled, true, + "a run resets the app and takes the one device session the recording is being read through"); + // Save lives in the Test Description section, which starts folded: saving half a flow the agent + // is still writing is a file the user did not ask for. + await act(async () => { (dom.window.document.querySelector("[data-testing-description-toggle]") as HTMLElement).click(); }); + assert.equal((dom.window.document.querySelector("[data-testing-save]") as HTMLButtonElement).disabled, true); + + Object.assign(agent, { active: false, status: "finished", revision: 2, note: "4 landmarks" }); + await act(async () => { await (panel as any).readAgentState(); }); + assert.equal(banner(), null); + // "Recorded" reads as "saved" to anyone who did not watch it happen. + assert.match(dom.window.document.querySelector("[data-testing-agent-handover]")?.textContent ?? "", + /The agent finished recording 1 step — 4 landmarks\. Nothing has been saved or run/); +}); diff --git a/packages/core/test/testing-agent.test.ts b/packages/core/test/testing-agent.test.ts new file mode 100644 index 00000000..8f6d4c50 --- /dev/null +++ b/packages/core/test/testing-agent.test.ts @@ -0,0 +1,406 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { TestingAgentRecordings, registerTestingAgentTools } from "../src/testing-agent.ts"; +import type { TestingDeviceAction, TestingDeviceTarget, TestingDeviceTree } from "../src/testing-device.ts"; +import type { TestingDevices } from "../src/testing-device.ts"; +import { TestDraftStore } from "../src/testing-drafts.ts"; +import { mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { udidSchema } from "../../shared/src/protocol.ts"; +import { deviceIdSchema, isRecordableDevice } from "../../shared/src/panel-target.ts"; + +const SCREEN = { width: 402, height: 874 }; +const CONTEXT = { projectPath: "/project", deviceId: "11111111-2222-3333-4444-555555555555", platform: "ios" as const, bundleId: "com.example.app" }; + +type Node = { + nodeId: string; parentId?: string; role?: string; label?: string; text?: string; value?: string; + placeholder?: string; identifier?: string; hidden?: boolean; disabled?: boolean; selected?: boolean; + bounds?: { x: number; y: number; width: number; height: number }; +}; + +/** One element as the runner's projection reports it, plus the agent-device ref a step lands on. */ +function tree(nodes: Node[]): TestingDeviceTree { + return { + screen: SCREEN, + tree: nodes.map((node, index) => ({ + name: node.label ?? node.text ?? node.nodeId, + bounds: node.bounds ?? { x: 0, y: 100 + index * 40, width: 300, height: 40 }, + depth: node.parentId ? 1 : 0, + source: "accessibility" as const, + ...node, + })), + refs: Object.fromEntries(nodes.map(node => [node.nodeId, `e-${node.nodeId}`])), + }; +} + +/** The device, recording what an agent recording asked it to do without talking to a simulator. */ +function fakeDevices(screens: TestingDeviceTree[]) { + const actions: { target: TestingDeviceTarget; action: TestingDeviceAction }[] = []; + let reads = 0; + const devices = { + read: async () => screens[Math.min(reads++, screens.length - 1)], + perform: async (target: TestingDeviceTarget, action: TestingDeviceAction) => { actions.push({ target, action }); }, + release: async () => undefined, + } as unknown as TestingDevices; + return { devices, actions, reads: () => reads }; +} + +const EXPLORE = tree([ + { nodeId: "bar", role: "tablist", label: "Tab Bar", bounds: { x: 0, y: 800, width: 402, height: 70 } }, + { nodeId: "tab", parentId: "bar", role: "tab", label: "Explore", selected: true, bounds: { x: 100, y: 800, width: 100, height: 70 } }, + { nodeId: "cta", role: "button", label: "Go To Explore List", text: "Go To Explore List" }, + { nodeId: "blank", role: "other" }, + { nodeId: "gone", role: "button", label: "Hidden", hidden: true }, +]); + +const DETAIL = tree([ + { nodeId: "heat", role: "text", label: "Live activity heatmap" }, + { nodeId: "search", role: "searchbox", placeholder: "Search reviews", value: "" }, + { nodeId: "back", role: "button", identifier: "BackButton", label: "Back" }, +]); + +async function started(screens: TestingDeviceTree[] = [EXPLORE]) { + const device = fakeDevices(screens); + // A real store on a temporary directory: finishing keeps the flow, and the test must not write + // into the user's own Application Support folder to prove it. + const drafts = new TestDraftStore(await mkdtemp(join(tmpdir(), "mobile-dev-agent-drafts-"))); + const recordings = new TestingAgentRecordings(device.devices, drafts); + recordings.setContext(CONTEXT); + recordings.begin({ title: "the explore details screen", goal: "cover the Explore Details landmarks" }); + return { ...device, recordings, drafts }; +} + +test("a recording refuses to start when the panel has nothing selected", async () => { + const { devices } = fakeDevices([EXPLORE]); + const recordings = new TestingAgentRecordings(devices); + // No project and no device is not something to pick for the user: the project picker is an OS + // folder dialog, which must not open inside an agent's turn. + assert.throws(() => recordings.begin({ title: "t", goal: "g" }), /an e2e project, no a booted simulator/); + recordings.setContext({ ...CONTEXT, bundleId: "" }); + assert.throws(() => recordings.begin({ title: "t", goal: "g" }), /bundle id/); + assert.equal(recordings.state().status, "idle"); +}); + +test("a second recording is refused rather than stomping on the first", async () => { + const { recordings } = await started(); + assert.throws(() => recordings.begin({ title: "another", goal: "g" }), /already in flight/); + assert.equal(recordings.state().title, "the explore details screen"); +}); + +test("a screen read answers with the selector the recorder would emit, and its rank", async () => { + const { recordings } = await started(); + const screen = await recordings.screen(); + const rows = Object.fromEntries(screen.elements.map(row => [row.name ?? row.testId, row])); + // The agent never writes a locator; it reads the one the recorder resolved and names a ref. + assert.equal(rows["Explore"].selector, 'screen.getByRole("tab", "Explore")'); + assert.equal(rows["Explore"].rank, "role + name"); + assert.equal(rows["Explore"].unique, true); + assert.equal(rows["Explore"].selected, true); + assert.equal(rows["Go To Explore List"].selector, 'screen.getByRole("button", "Go To Explore List")'); + // An off-screen node is still something to assert about — `getByText` resolves it either way — + // so it stays in the read, flagged rather than dropped. + assert.equal(rows["Hidden"].offScreen, true); + assert.equal(rows["Explore"].offScreen, undefined); + // Nothing names the bare container, and the read says so by carrying no selector for it. + assert.equal(screen.elements.find(row => row.ref.endsWith("-blank"))?.selector, undefined); + assert.deepEqual(screen.screen, SCREEN); +}); + +test("a step names the element by ref and is taken on the device", async () => { + const { recordings, actions } = await started(); + const screen = await recordings.screen(); + const tab = screen.elements.find(row => row.name === "Explore")!; + const result = await recordings.step({ ref: tab.ref, kind: "tap" }); + + assert.equal(result.acted, true); + assert.equal(result.source, 'screen.getByRole("tab", "Explore").tap()'); + // The step is placed on the element's own agent-device ref, which is where the runner's locator + // action will land — not on a coordinate computed from a rectangle. + assert.deepEqual(actions.map(entry => entry.action), [{ kind: "tap", ref: "e-tab" }]); + assert.deepEqual(actions[0].target, { deviceId: CONTEXT.deviceId, platform: "ios", projectPath: CONTEXT.projectPath, bundleId: CONTEXT.bundleId }); + const [step] = recordings.state().steps; + assert.equal(step.label, 'Tap "Explore" (tab)'); +}); + +test("an assertion records without touching the device", async () => { + const { recordings, actions } = await started(); + const screen = await recordings.screen(); + const cta = screen.elements.find(row => row.name === "Go To Explore List")!; + const result = await recordings.step({ ref: cta.ref, kind: "assertVisible" }); + assert.equal(result.acted, false); + assert.equal(result.source, 'expect(screen.getByRole("button", "Go To Explore List")).toBeVisible()'); + assert.deepEqual(actions, [], "an assertion reads the screen and changes nothing"); +}); + +test("a ref from an earlier read is refused rather than acted on", async () => { + const { recordings } = await started([EXPLORE, DETAIL]); + const screen = await recordings.screen(); + const tab = screen.elements.find(row => row.name === "Explore")!; + await recordings.step({ ref: tab.ref, kind: "tap" }); + // Every action invalidates the tree, so the refs it was read with go with it. + await assert.rejects(recordings.step({ ref: tab.ref, kind: "tap" }), /not from the latest screen read/); + const next = await recordings.screen(); + assert.equal(next.elements.some(row => row.ref === tab.ref), false, "a new read mints new refs"); +}); + +test("a fill types the value through agent-device and the step emits it", async () => { + const { recordings, actions } = await started([DETAIL]); + const screen = await recordings.screen(); + const search = screen.elements.find(row => row.placeholder === "Search reviews")!; + assert.equal(search.selector, 'screen.getByPlaceholder("Search reviews")'); + const result = await recordings.step({ ref: search.ref, kind: "fill", value: "Roaming Rosa" }); + assert.equal(result.source, 'screen.getByPlaceholder("Search reviews").fill("Roaming Rosa")'); + assert.deepEqual(actions.map(entry => entry.action), [{ kind: "type", ref: "e-search", text: "Roaming Rosa" }]); +}); + +test("a step that needs text is refused without it", async () => { + const { recordings } = await started([DETAIL]); + const screen = await recordings.screen(); + const heat = screen.elements.find(row => row.name === "Live activity heatmap")!; + await assert.rejects(recordings.step({ ref: heat.ref, kind: "assertText" }), /needs a value/); + await assert.rejects(recordings.step({ ref: heat.ref, kind: "selectOption" }), /not a step the runner has/); +}); + +test("an element nothing can name is pointed at the sentence instead of a coordinate", async () => { + const { recordings } = await started(); + const screen = await recordings.screen(); + const blank = screen.elements.find(row => row.ref.endsWith("-blank"))!; + // The one deliberate difference from the mirror: a tap falls back to `screen.tapAt`, and an + // agent does not, because it can describe what it cannot name. + await assert.rejects(recordings.step({ ref: blank.ref, kind: "tap" }), /Record it as a sentence instead/); +}); + +test("a sentence records an agent step and nothing is sent to the device", async () => { + const { recordings, actions } = await started(); + const result = await recordings.step({ instruction: "dismiss the onboarding sheet" }); + assert.equal(result.source, 'agent.act("dismiss the onboarding sheet")'); + assert.deepEqual(actions, []); + assert.equal(recordings.state().steps[0].kind, "agentAct"); + const judged = await recordings.step({ kind: "agentAssert", instruction: "the sheet is gone" }); + assert.equal(judged.source, 'agent.assert("the sheet is gone")'); +}); + +test("a swipe with no element swipes the viewport, well inside its edges", async () => { + const { recordings, actions } = await started(); + await recordings.screen(); + const result = await recordings.step({ kind: "swipe", direction: "up" }); + assert.equal(result.source, 'screen.swipe({ direction: "up" })'); + // A swipe is as actionable as a tap, so it carries something on the screen to wait for. + assert.ok((recordings.state().steps[0] as any).anchor?.kind, "something on the screen to wait for"); + // The runner performs `screen.swipe` as agent-device's own `scroll`, so the recorder does too: + // a trajectory of our own falls outside the viewport the moment the keyboard shrinks it. + assert.deepEqual(actions[0].action, { kind: "scroll", direction: "up" }); + await assert.rejects(recordings.step({ kind: "tap" }), /needs a ref/); +}); + +test("the revision moves on every change, which is what the panel adopts on", async () => { + const { recordings } = await started(); + const begun = recordings.state().revision; + const screen = await recordings.screen(); + assert.equal(recordings.state().revision, begun, "reading the screen records nothing"); + await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "assertSelected" }); + assert.equal(recordings.state().revision, begun + 1); + await recordings.finish({ note: "covered the tab bar" }); + assert.equal(recordings.state().revision, begun + 2); +}); + +test("finishing leaves the steps dirty in the panel and saves nothing", async () => { + const { recordings } = await started(); + const screen = await recordings.screen(); + await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "tap" }); + const result = await recordings.finish({ note: "the Explore tab" }); + + assert.equal(result.steps, 1); + // The whole file, emitted by the same emitter the panel's preview and Save use. + assert.match(result.source, /import \{ test \} from "@e2e-dev\/mobile";/); + assert.match(result.source, /await screen\.getByRole\("tab", "Explore"\)\.tap\(\);/); + assert.match(result.next, /unsaved/); + const state = recordings.state(); + assert.equal(state.active, false); + assert.equal(state.status, "finished"); + // The steps stay: they are the user's to adjust and save. + assert.equal(state.steps.length, 1); + await assert.rejects(recordings.screen(), /No agent recording is in flight/); +}); + +test("aborting keeps what was recorded and lets the next recording start", async () => { + const { recordings } = await started(); + const screen = await recordings.screen(); + await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "assertVisible" }); + assert.deepEqual(await recordings.abort({ reason: "the user pressed Stop" }), { ok: true, steps: 1, wasRecording: true }); + assert.equal(recordings.state().status, "aborted"); + assert.equal(recordings.state().steps.length, 1, "whatever was recorded is the user's to keep"); + // Aborting an idle recorder is not an error; the panel's Stop button can always be pressed. + assert.deepEqual(await recordings.abort({ reason: "again" }), { ok: true, steps: 1, wasRecording: false }); + recordings.begin({ title: "another", goal: "g" }); + assert.deepEqual(recordings.state().steps, []); +}); + +test("a selection the panel cannot act on clears the context rather than half-setting it", () => { + const { devices } = fakeDevices([EXPLORE]); + const recordings = new TestingAgentRecordings(devices); + const server = { registerTool: () => undefined, registerResource: () => undefined } as never; + registerTestingAgentTools(server, devices, recordings); + recordings.setContext(CONTEXT); + recordings.clearContext(); + assert.throws(() => recordings.begin({ title: "t", goal: "g" }), /Testing tab/); +}); + +test("the five model tools are visible to the model and the panel's two are not", () => { + const registered: { name: string; config: Record<string, any> }[] = []; + const server = { + registerTool: (name: string, config: Record<string, any>) => { registered.push({ name, config }); }, + registerResource: () => undefined, + } as never; + const { devices } = fakeDevices([EXPLORE]); + registerTestingAgentTools(server, devices); + const visibility = Object.fromEntries(registered.map(entry => [entry.name, entry.config._meta?.ui?.visibility])); + // A narrow, intention-shaped surface is what keeps write_file and run_test out of the model's + // hands; see the comment at the top of testing-tools.ts. + for (const name of ["begin", "screen", "step", "finish", "abort"]) { + assert.deepEqual(visibility[`mobile_testing_agent_${name}`], ["app", "model"], name); + } + assert.deepEqual(visibility["mobile_testing_agent_state"], ["app"]); + assert.deepEqual(visibility["mobile_testing_set_context"], ["app"]); +}); + +/** The Explore List, which shares nothing with the Explore screen but the application itself. */ +const LIST = tree([ + { nodeId: "nav", role: "navigation", identifier: "explore-list" }, + { nodeId: "title", role: "text", label: "Explore Cities" }, + { nodeId: "search", role: "searchbox", placeholder: "Search transactions" }, +]); + +/** The Explore screen with a disclosure opened: everything it had, plus what the row revealed. */ +const EXPANDED = tree([ + { nodeId: "bar", role: "tablist", label: "Tab Bar", bounds: { x: 0, y: 800, width: 402, height: 70 } }, + { nodeId: "tab", parentId: "bar", role: "tab", label: "Explore", selected: true, bounds: { x: 100, y: 800, width: 100, height: 70 } }, + { nodeId: "cta", role: "button", label: "Go To Explore List", text: "Go To Explore List" }, + { nodeId: "blank", role: "other" }, + { nodeId: "gone", role: "button", label: "Hidden", hidden: true }, + { nodeId: "revealed", role: "text", label: "This app has two screens" }, +]); + +test("a step that replaces the screen says so, with the recording's own scope", async () => { + const { recordings } = await started([EXPLORE, LIST]); + const screen = await recordings.screen(); + const cta = screen.elements.find(row => row.name === "Go To Explore List")!; + const result = await recordings.step({ ref: cta.ref, kind: "tap" }); + + assert.equal(result.navigated, true); + // The goal comes back with it, because the thing the agent has to weigh is what it was asked to + // record — a screen name is a boundary, and it only knows that from the sentence it wrote. + assert.equal(result.goal, "cover the Explore Details landmarks"); + assert.match(result.next!, /assert that you arrived and call mobile_testing_agent_finish/); +}); + +test("a step that only changes the screen is not a navigation", async () => { + const { recordings } = await started([EXPLORE, EXPANDED]); + const screen = await recordings.screen(); + const cta = screen.elements.find(row => row.name === "Go To Explore List")!; + const result = await recordings.step({ ref: cta.ref, kind: "tap" }); + + // Opening a disclosure leaves the screen in place and adds to it. A hint that fired here would + // end every flow at its first interesting step. + assert.equal(result.navigated, undefined); + assert.match(result.next!, /Call mobile_testing_agent_screen before the next step/); +}); + +test("the hint is advisory: the step is recorded and the recording goes on", async () => { + const { recordings } = await started([EXPLORE, LIST]); + const screen = await recordings.screen(); + await recordings.step({ ref: screen.elements.find(row => row.name === "Go To Explore List")!.ref, kind: "tap" }); + + // Nothing refuses, nothing counts, nothing stops: a checkout and a sign-in cross screens on + // purpose, and a recorder that would not let them is worse than one that over-records. + assert.equal(recordings.state().steps.length, 1); + const next = await recordings.screen(); + const title = next.elements.find(row => row.name === "Explore Cities")!; + const arrived = await recordings.step({ ref: title.ref, kind: "assertVisible" }); + assert.equal(arrived.ok, true); + assert.equal(recordings.state().steps.length, 2); +}); + +test("the read a step already took is the read the next screen call answers with", async () => { + const device = fakeDevices([EXPLORE, LIST]); + const recordings = new TestingAgentRecordings(device.devices); + recordings.setContext(CONTEXT); + recordings.begin({ title: "t", goal: "g" }); + + const screen = await recordings.screen(); + assert.equal(device.reads(), 1); + await recordings.step({ ref: screen.elements.find(row => row.name === "Go To Explore List")!.ref, kind: "tap" }); + // The step had to read to answer whether it navigated; the agent's own re-read is that read. + assert.equal(device.reads(), 2); + const after = await recordings.screen(); + assert.equal(device.reads(), 2, "one snapshot per step, not two"); + assert.ok(after.elements.some(row => row.name === "Explore Cities")); + // And the refs it hands out are the ones the step's own read minted, so they are current. + assert.equal(after.readId, 2); +}); + +test("arriving at the screen the recording is for is not leaving it", async () => { + // A tab switch keeps the tab bar and the chrome around it — about two fifths of the read on a + // real app — and it is usually the navigation *to* the named screen. A boundary warning there + // would land exactly as the agent arrives. + const HOME = tree([ + { nodeId: "bar", role: "tablist", label: "Tab Bar", bounds: { x: 0, y: 800, width: 402, height: 70 } }, + { nodeId: "home", parentId: "bar", role: "tab", label: "Home", selected: true, bounds: { x: 0, y: 800, width: 100, height: 70 } }, + { nodeId: "tab", parentId: "bar", role: "tab", label: "Explore", bounds: { x: 100, y: 800, width: 100, height: 70 } }, + { nodeId: "settings", parentId: "bar", role: "tab", label: "Settings", bounds: { x: 200, y: 800, width: 100, height: 70 } }, + { nodeId: "welcome", role: "text", label: "Welcome to Expo" }, + { nodeId: "started", role: "text", label: "GET STARTED" }, + ]); + const { recordings } = await started([HOME, EXPLORE]); + const screen = await recordings.screen(); + const result = await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "tap" }); + assert.equal(result.navigated, undefined); +}); + +test("finishing keeps the flow, so a panel that was never looking still finds it", async () => { + const { recordings, drafts } = await started(); + const screen = await recordings.screen(); + await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "tap" }); + await recordings.finish({ note: "recorded the Explore tab" }); + + const kept = await drafts.list(CONTEXT.projectPath); + assert.equal(kept.length, 1); + assert.equal(kept[0].origin, "agent"); + assert.equal(kept[0].title, "the explore details screen"); + assert.equal(kept[0].stepCount, 1); + // The panel adopts this id rather than keeping a second copy of the same steps a moment later. + assert.equal(recordings.state().draftId, kept[0].id); +}); + +test("an abandoned recording is kept too: the steps are the user's to keep", async () => { + const { recordings, drafts } = await started(); + const screen = await recordings.screen(); + await recordings.step({ ref: screen.elements.find(row => row.name === "Explore")!.ref, kind: "tap" }); + await recordings.abort({ reason: "the user pressed Stop" }); + + assert.equal((await drafts.list(CONTEXT.projectPath)).length, 1); +}); + +test("a recording with no steps keeps nothing", async () => { + const { recordings, drafts } = await started(); + await recordings.finish({}); + assert.deepEqual(await drafts.list(CONTEXT.projectPath), []); + assert.equal(recordings.state().draftId, ""); +}); + +test("an adb serial is a valid device id where a uuid schema refuses it", () => { + assert.equal(udidSchema.safeParse("emulator-5554").success, false); + assert.equal(deviceIdSchema.safeParse("emulator-5554").success, true); + assert.equal(deviceIdSchema.safeParse(CONTEXT.deviceId).success, true); +}); + +test("only a booted simulator or emulator is recordable", () => { + const base = { udid: "x", name: "d", runtime: "" }; + assert.equal(isRecordableDevice({ ...base, state: "Booted", platform: "ios", kind: "simulator" }), true); + assert.equal(isRecordableDevice({ ...base, udid: "emulator-5554", state: "Booted", platform: "android", kind: "emulator" }), true); + assert.equal(isRecordableDevice({ ...base, state: "Shutdown", platform: "android", kind: "emulator" }), false); + assert.equal(isRecordableDevice({ ...base, state: "connected", platform: "android", kind: "physical" } as never), false); + assert.equal(isRecordableDevice({ ...base, state: "connected", platform: "ios", kind: "physical" } as never), false); + assert.equal(isRecordableDevice(undefined), false); +}); diff --git a/packages/core/test/testing-device.test.ts b/packages/core/test/testing-device.test.ts new file mode 100644 index 00000000..605176a4 --- /dev/null +++ b/packages/core/test/testing-device.test.ts @@ -0,0 +1,316 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { TestingDevices, projectedComponents } from "../src/testing-device.ts"; +import { registerTestingTools } from "../src/testing-tools.ts"; + +/** + * Stands in for `@e2e-dev/mobile`'s `projectSnapshot`. The real projection is the project's own + * build and is exercised against a device in the recorded-flow check; what matters here is that + * whatever it says about an element reaches the panel unchanged. + */ +const projection = { + projectSnapshot(raw: readonly unknown[], { mintId }: { mintId: () => string }) { + const index = (raw as Record<string, unknown>[]).map(node => ({ + id: mintId(), kind: String(node.kind ?? ""), ref: node.ref as string | undefined, + node: node.node as Record<string, unknown>, parent: undefined, + })); + for (const [position, entry] of index.entries()) { + const parent = (raw as Record<string, unknown>[])[position].parent; + if (typeof parent === "number") (entry as { parent?: unknown }).parent = index[parent]; + } + return { index, viewport: { width: 402, height: 874 } } as never; + }, +}; + +const rect = { x: 24, y: 476, width: 354, height: 35 }; + +test("the runner's own fields reach the panel untouched", () => { + const { tree, screen } = projectedComponents(projection as never, [ + { kind: "tab-bar", node: { role: "tablist", name: "Tab Bar", text: "Tab Bar", rect } }, + { kind: "button", parent: 0, node: { role: "tab", name: "Explore", text: "Explore", rect } }, + ]); + assert.deepEqual(screen, { width: 402, height: 874 }); + // `role` above all: the engine said `tab` for a button inside a tab bar, and nothing renames it. + assert.equal(tree[1].role, "tab"); + assert.equal(tree[1].label, "Explore", "the runner's `name` is what getByRole and getByLabel match"); + assert.equal(tree[1].text, "Explore"); + assert.equal(tree[1].parentId, tree[0].nodeId, "the parent link the innermost-only rule needs"); + assert.equal(tree[1].depth, 1); +}); + +test("a placeholder and a hidden state survive the projection", () => { + const { tree } = projectedComponents(projection as never, [ + { kind: "text-field", node: { role: "textbox", attributes: { placeholder: "Search transactions" }, value: "Search transactions", rect } }, + { kind: "static-text", node: { role: "text", name: "Gone", states: { hidden: true }, rect } }, + ]); + assert.equal(tree[0].placeholder, "Search transactions"); + assert.equal(tree[0].label, undefined, "a field named only by its placeholder has no accessible name"); + // The name shown in the panel still has to say something, so it falls back to the placeholder. + assert.equal(tree[0].name, "Search transactions"); + assert.equal(tree[1].hidden, true); +}); + +test("an element with no box is not something a tap could land on", () => { + const { tree } = projectedComponents(projection as never, [ + { kind: "other", node: { role: "other", name: "Nowhere" } }, + { kind: "other", node: { role: "other", name: "Empty", rect: { x: 0, y: 0, width: 0, height: 10 } } }, + { kind: "text", node: { role: "text", name: "Here", rect } }, + ]); + assert.deepEqual(tree.map(item => item.label), ["Here"]); +}); + +/** A client that records the calls made on it, so the attach options can be asserted. */ +function fakeAgentDevice(calls: Record<string, unknown>[], snapshot: () => { nodes?: unknown }) { + return async () => ({ + createAgentDeviceClient: () => ({ + apps: { open: async (options: Record<string, unknown>) => { calls.push({ call: "open", ...options }); } }, + capture: { snapshot: async (options: Record<string, unknown>) => { calls.push({ call: "snapshot", ...options }); return snapshot(); } }, + sessions: { close: async (options: Record<string, unknown>) => { calls.push({ call: "close", ...options }); } }, + interactions: { + scroll: async (options: Record<string, unknown>) => { calls.push({ call: "scroll", ...options }); }, + click: async (options: Record<string, unknown>) => { calls.push({ call: "click", ...options }); }, + fill: async (options: Record<string, unknown>) => { calls.push({ call: "fill", ...options }); }, + swipe: async (options: Record<string, unknown>) => { calls.push({ call: "swipe", ...options }); }, + }, + }), + }); +} + +const target = { deviceId: "11111111-2222-3333-4444-555555555555", platform: "ios" as const, projectPath: "/project", bundleId: "com.example.app" }; + +test("the recorder attaches to the running app and never relaunches it", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + // The projection is the project's; stub it so the test is about the session, not about roles. + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + + const open = calls.find(entry => entry.call === "open")!; + // Relaunching would throw away the screen the user navigated to, which is the whole point. + assert.equal(open.relaunch, false); + assert.equal(open.app, "com.example.app"); + assert.equal(open.udid, target.deviceId); + // `--foreground` refuses outright when more than one app is running, which is every real simulator. + assert.equal(open.foreground, undefined); + // The runner's own snapshot options: a different tree would resolve differently. + assert.equal(calls.find(entry => entry.call === "snapshot")!.interactiveOnly, false); +}); + +test("a second read reuses the session rather than attaching again", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + await devices.read(target); + + assert.equal(calls.filter(entry => entry.call === "open").length, 1); + assert.equal(calls.filter(entry => entry.call === "snapshot").length, 2); +}); + +test("a session that went away is reattached once", async () => { + const calls: Record<string, unknown>[] = []; + let first = true; + const devices = new TestingDevices(fakeAgentDevice(calls, () => { + if (first) { first = false; throw new Error("iOS snapshot requires an active app session on the target device."); } + return { nodes: [] }; + }) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + + assert.equal(calls.filter(entry => entry.call === "open").length, 2, "the app was closed under the panel; attach again"); +}); + +test("releasing closes the session, so the next e2e run is not refused the device", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + await devices.release(); + + assert.equal(calls.filter(entry => entry.call === "close").length, 1); + // Nothing is held any more, so releasing again is not a second close. + await devices.release(); + assert.equal(calls.filter(entry => entry.call === "close").length, 1); + + // And the next read attaches from scratch. + await devices.read(target); + assert.equal(calls.filter(entry => entry.call === "open").length, 2); +}); + +test("a failed read still leaves the device releasable", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => { throw new Error("the runner fell over"); }) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await assert.rejects(devices.read(target), /fell over/); + await devices.release(); + assert.equal(calls.filter(entry => entry.call === "close").length, 1); +}); + +test("a project without the mobile engine says so instead of guessing at roles", async () => { + const devices = new TestingDevices(fakeAgentDevice([], () => ({ nodes: [] })) as never); + await assert.rejects( + devices.read({ ...target, projectPath: "/does/not/exist" }), + /@e2e-dev\/mobile/, + ); +}); + +test("shutting the plugin down gives the recorder's device back", async () => { + // A server that goes away without releasing leaves the claim behind, and the agent-device daemon + // that holds it outlives the server. The user's own `e2e run` then refuses to boot the device + // with "has an ownership claim that could not be verified", which points at nothing they did. + const closed: unknown[] = []; + const devices = new TestingDevices(async () => ({ + createAgentDeviceClient: () => ({ + apps: { open: async () => undefined }, + capture: { snapshot: async () => ({ nodes: [] }) }, + sessions: { close: async (options?: unknown) => { closed.push(options); } }, + interactions: { click: async () => undefined, fill: async () => undefined, swipe: async () => undefined, scroll: async () => undefined }, + }), + }) as never); + const server = { registerTool: () => undefined, registerResource: () => undefined } as never; + const projects = { list: async () => [], choose: async () => undefined, record: async () => undefined } as never; + const shutdown = registerTestingTools(server, projects, devices); + // Nothing is attached, so there is nothing to give back and nothing is called. + await shutdown(); + assert.deepEqual(closed, []); + (devices as unknown as { attached?: string; client?: unknown }).attached = "ios:udid"; + (devices as unknown as { client?: unknown }).client = (await (devices as unknown as { connect(): Promise<unknown> }).connect()); + await shutdown(); + assert.deepEqual(closed, [{ session: "mobile-dev-recorder" }]); +}); + +test("each element keeps the agent-device ref an action is placed on", () => { + // The panel never needs these — it records selectors and drives the mirror — but an agent + // recording acts through agent-device, whose tap and fill take a ref. Keeping the pair beside + // the projection is what stops the server resolving the same element a second time. + const { tree, refs } = projectedComponents(projection as never, [ + { kind: "button", ref: "e1", node: { role: "tab", name: "Explore", rect } }, + { kind: "other", ref: "e2", node: { role: "other", name: "Nowhere" } }, + ]); + // The `@` the projection stripped is put back: `click({ ref: "e1" })` parses a bare ref as a + // selector and refuses it with `Invalid selector term "e1", expected key=value`. + assert.deepEqual(refs, { [tree[0].nodeId!]: "@e1" }, "an element with no box is not in the tree, so it has no ref either"); +}); + +test("disabled and selected survive the projection, for the matchers that read them", () => { + const { tree } = projectedComponents(projection as never, [ + { kind: "button", node: { role: "tab", name: "Explore", states: { selected: true }, rect } }, + { kind: "button", node: { role: "button", name: "Save", states: { disabled: true }, rect } }, + ]); + assert.equal(tree[0].selected, true); + assert.equal(tree[0].disabled, undefined); + assert.equal(tree[1].disabled, true); +}); + +test("an agent recording's step is placed on the element's own ref, on the recorder's session", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + await devices.perform(target, { kind: "tap", ref: "e1" }); + await devices.perform(target, { kind: "doubleTap", ref: "e1" }); + await devices.perform(target, { kind: "type", ref: "e2", text: "Roaming Rosa" }); + await devices.perform(target, { kind: "swipe", from: { x: 1, y: 2 }, to: { x: 3, y: 4 } }); + + const clicks = calls.filter(entry => entry.call === "click"); + assert.deepEqual(clicks.map(entry => entry.ref), ["e1", "e1"]); + // One session, the recorder's: an agent recording is another holder of it, never a second one. + assert.ok(clicks.every(entry => entry.session === "mobile-dev-recorder" && entry.udid === target.deviceId)); + assert.equal(clicks[0].doubleTap, undefined); + // The next step is resolved against the screen this one produced, so the wait for that screen + // to stop moving belongs here; without it agent-device refuses the next action outright. + assert.equal(clicks[0].settle, true); + assert.equal(calls.find(entry => entry.call === "fill")!.settle, true); + assert.equal(calls.find(entry => entry.call === "swipe")!.settle, undefined, "a swipe has no element to come to rest"); + assert.equal(clicks[1].doubleTap, true, "agent-device takes the double tap as the one gesture the step emits"); + const fill = calls.find(entry => entry.call === "fill")!; + assert.equal(fill.text, "Roaming Rosa"); + assert.deepEqual(calls.find(entry => entry.call === "swipe")!.from, { x: 1, y: 2 }); + // Nothing re-attached: the step runs on the session the read already opened. + assert.equal(calls.filter(entry => entry.call === "open").length, 1); +}); + +test("a step on a device nothing is attached to attaches first", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + await devices.perform(target, { kind: "tap", ref: "e1" }); + assert.equal(calls.filter(entry => entry.call === "open").length, 1); + assert.equal(calls.find(entry => entry.call === "open")!.relaunch, false); +}); + +test("an Android emulator is addressed by serial, never udid, on every call", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + const android = { ...target, deviceId: "emulator-5554", platform: "android" as const }; + + await devices.read(android); + await devices.perform(android, { kind: "tap", ref: "e1" }); + await devices.perform(android, { kind: "type", ref: "e2", text: "hi" }); + await devices.perform(android, { kind: "scroll", direction: "down" }); + await devices.perform(android, { kind: "swipe", from: { x: 1, y: 2 }, to: { x: 3, y: 4 } }); + + const names = ["open", "snapshot", "click", "fill", "scroll", "swipe"]; + for (const call of names) { + const entry = calls.find(item => item.call === call)!; + assert.ok(entry, call); + assert.equal(entry.platform, "android", call); + assert.equal(entry.serial, "emulator-5554", call); + assert.equal(entry.udid, undefined, call); + } + assert.equal(calls.find(item => item.call === "open")!.relaunch, false); +}); + +test("the same id on another platform is not a cached attach", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + await devices.read({ ...target, platform: "android" }); + + assert.equal(calls.filter(entry => entry.call === "open").length, 2); +}); + +test("changing project or bundle on one device is not a cached attach", async () => { + const calls: Record<string, unknown>[] = []; + const devices = new TestingDevices(fakeAgentDevice(calls, () => ({ nodes: [] })) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/other-project", Promise.resolve(projection)); + + await devices.read(target); + await devices.read({ ...target, bundleId: "com.example.other" }); + await devices.read({ ...target, projectPath: "/other-project" }); + + assert.equal(calls.filter(entry => entry.call === "open").length, 3); +}); + +test("release finishes before a later read can attach a new session", async () => { + const calls: Record<string, unknown>[] = []; + let finishClose!: () => void; + const close = new Promise<void>(resolve => { finishClose = resolve; }); + const devices = new TestingDevices(async () => ({ + createAgentDeviceClient: () => ({ + apps: { open: async (options: Record<string, unknown>) => { calls.push({ call: "open", ...options }); } }, + capture: { snapshot: async () => ({ nodes: [] }) }, + sessions: { close: async () => { calls.push({ call: "close" }); await close; } }, + interactions: { click: async () => {}, fill: async () => {}, swipe: async () => {}, scroll: async () => {} }, + }), + }) as never); + (devices as unknown as { projections: Map<string, unknown> }).projections.set("/project", Promise.resolve(projection)); + + await devices.read(target); + const releasing = devices.release(); + const reading = devices.read({ ...target, bundleId: "com.example.next" }); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.equal(calls.filter(entry => entry.call === "open").length, 1, "the new attach waits for close"); + finishClose(); + await Promise.all([releasing, reading]); + assert.deepEqual(calls.map(entry => entry.call), ["open", "close", "open"]); +}); diff --git a/packages/core/test/testing-drafts.test.ts b/packages/core/test/testing-drafts.test.ts new file mode 100644 index 00000000..006c7506 --- /dev/null +++ b/packages/core/test/testing-drafts.test.ts @@ -0,0 +1,120 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { TestDraftStore } from "../src/testing-drafts.ts"; +import type { RecordedStep } from "@mobile-dev/shared/test-recording.ts"; + +const PROJECT = "/Users/someone/projects/expoapp"; + +function steps(count: number): RecordedStep[] { + return Array.from({ length: count }, (unused, index) => ({ + id: `step-${index}`, + label: `Tap "Explore ${index}" (tab)`, + kind: "tap" as const, + selector: { kind: "role" as const, role: "tab", name: `Explore ${index}` }, + })); +} + +async function store() { + return new TestDraftStore(await mkdtemp(join(tmpdir(), "mobile-dev-drafts-"))); +} + +test("keeps a recording and reads its steps back", async () => { + const drafts = await store(); + const saved = await drafts.save({ projectPath: PROJECT, title: "the explore screen", steps: steps(3) }); + assert.equal(saved.steps.length, 3); + const read = await drafts.read(saved.id); + assert.deepEqual(read?.steps, steps(3)); + assert.equal(read?.title, "the explore screen"); + assert.equal(read?.origin, "user"); +}); + +test("keeps edited source as the authority for a reopened draft", async () => { + const drafts = await store(); + const source = 'test("edited", async ({ screen }) => {\n await screen.getByText("Done").tap();\n});'; + const saved = await drafts.save({ projectPath: PROJECT, title: "edited", steps: steps(1), source }); + assert.equal((await drafts.read(saved.id))?.source, source); + assert.equal((await drafts.list(PROJECT))[0]?.stepCount, 1); +}); + +test("replaces a draft in place rather than listing it twice", async () => { + const drafts = await store(); + const first = await drafts.save({ projectPath: PROJECT, title: "a flow", steps: steps(1) }); + const second = await drafts.save({ id: first.id, projectPath: PROJECT, title: "a flow", steps: steps(4) }); + assert.equal(second.id, first.id); + assert.equal(second.createdAt, first.createdAt); + const listed = await drafts.list(PROJECT); + assert.equal(listed.length, 1); + assert.equal(listed[0].stepCount, 4); +}); + +test("lists one project's drafts, newest first, and leaves another project's alone", async () => { + const drafts = await store(); + const old = await drafts.save({ projectPath: PROJECT, title: "older", steps: steps(1) }); + // Written second, so it is the newer of the two whatever the clock's resolution is. + await new Promise(resolve => setTimeout(resolve, 5)); + const recent = await drafts.save({ projectPath: PROJECT, title: "newer", steps: steps(2) }); + await drafts.save({ projectPath: "/Users/someone/projects/other", title: "elsewhere", steps: steps(1) }); + const listed = await drafts.list(PROJECT); + assert.deepEqual(listed.map(entry => entry.id), [recent.id, old.id]); + assert.equal((await drafts.list()).length, 3); +}); + +test("a trailing slash is the same project", async () => { + const drafts = await store(); + await drafts.save({ projectPath: `${PROJECT}/`, title: "a flow", steps: steps(1) }); + assert.equal((await drafts.list(PROJECT)).length, 1); +}); + +test("deleting one forgets it and says whether it was there", async () => { + const drafts = await store(); + const saved = await drafts.save({ projectPath: PROJECT, title: "a flow", steps: steps(1) }); + assert.equal(await drafts.remove(saved.id), true); + assert.equal(await drafts.remove(saved.id), false); + assert.equal(await drafts.read(saved.id), undefined); + assert.deepEqual(await drafts.list(PROJECT), []); +}); + +test("an unreadable file is skipped rather than taking the list down", async () => { + const drafts = await store(); + const saved = await drafts.save({ projectPath: PROJECT, title: "a flow", steps: steps(1) }); + await writeFile(join(drafts.directory, "99999999-9999-4999-8999-999999999999.json"), "{ truncated", "utf-8"); + const listed = await drafts.list(PROJECT); + assert.deepEqual(listed.map(entry => entry.id), [saved.id]); +}); + +test("a flow with no steps is refused, and a project path is required", async () => { + const drafts = await store(); + await assert.rejects(() => drafts.save({ projectPath: PROJECT, title: "empty", steps: [] }), /Record a step/); + await assert.rejects(() => drafts.save({ projectPath: " ", title: "no project", steps: steps(1) }), /project folder/); +}); + +test("the oldest drafts are dropped past the cap", async () => { + const drafts = await store(); + for (let index = 0; index < 33; index += 1) { + await drafts.save({ projectPath: PROJECT, title: `flow ${index}`, steps: steps(1) }); + await new Promise(resolve => setTimeout(resolve, 2)); + } + const files = (await readdir(drafts.directory)).filter(name => name.endsWith(".json")); + assert.equal(files.length, 30); + const listed = await drafts.list(PROJECT); + assert.equal(listed[0].title, "flow 32"); + assert.ok(!listed.some(entry => entry.title === "flow 0")); +}); + +test("an agent's steps round-trip, instructions and params included", async () => { + const drafts = await store(); + const flow: RecordedStep[] = [ + { id: "a", label: "Act", kind: "agentAct", instruction: "dismiss the onboarding sheet", params: { city: "Paris" } }, + { id: "b", label: "Assert", kind: "agentAssert", instruction: "the heatmap is shown", vision: "only" }, + { id: "c", label: "Fill", kind: "fill", selector: { kind: "placeholder", placeholder: "Search reviews" }, value: "quiet" }, + { id: "d", label: "Tap", kind: "tap", selector: { kind: "nth", inner: { kind: "text", text: "Paris" }, index: 1, matches: 2 } }, + { id: "e", label: "Swipe", kind: "swipe", selector: { kind: "point", point: { x: 10, y: 20 } }, direction: "left", + anchor: { kind: "testId", testId: "list" }, ambiguous: true, target: { role: "other", text: "Paris" } }, + ]; + const saved = await drafts.save({ projectPath: PROJECT, title: "agent flow", origin: "agent", steps: flow }); + assert.deepEqual((await drafts.read(saved.id))?.steps, flow); + assert.equal((await drafts.list(PROJECT))[0].origin, "agent"); +}); diff --git a/packages/core/test/testing-panel-ui.test.ts b/packages/core/test/testing-panel-ui.test.ts new file mode 100644 index 00000000..5b5b3f4a --- /dev/null +++ b/packages/core/test/testing-panel-ui.test.ts @@ -0,0 +1,943 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { renderEnvironment } from "./jsdom-env.ts"; + +const SCREEN = { width: 402, height: 874 }; +const VIEWPORT = { x: 10, y: 20, width: 300, height: 650, stageWidth: 320, stageHeight: 700 }; +// `useSyncExternalStore` compares snapshots by identity, so these are values, not fresh objects. +const ANNOTATIONS_SNAPSHOT = { viewport: VIEWPORT }; +const DEVICE_SNAPSHOT = { device: { udid: "udid", name: "iPhone 17 Pro", state: "Booted", platform: "ios", kind: "simulator" } }; + +/** One element as the e2e runner reports it: its own role, its accessible name in `label`. */ +const button = { + name: "Go To Explore List", label: "Go To Explore List", text: "Go To Explore List", + role: "button", depth: 1, source: "accessibility" as const, nodeId: "cta", + bounds: { x: 100, y: 400, width: 200, height: 44 }, +}; + +test("the element menu renders the runner's own steps over the mirror and records the one chosen", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-overlay", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const listeners = new Set<() => void>(); + const annotations = { + subscribe: (listener: () => void) => { listeners.add(listener); return () => { listeners.delete(listener); }; }, + getSnapshot: () => ANNOTATIONS_SNAPSHOT, + }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const menu = () => dom.window.document.querySelector("[data-element=recorder-menu]"); + const rows = () => [...dom.window.document.querySelectorAll("[data-recorder-offer]")] as HTMLButtonElement[]; + + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + assert.ok(menu(), "the right-click opens the menu over the outline"); + // The selector is named once and rows stay compact; exact code is still available as a tooltip. + assert.match(menu()?.textContent ?? "", /Selected element"Go To Explore List" \(button\)/); + assert.deepEqual(rows().map(row => row.textContent), [ + "Tap", + "Is Visible", + "Scroll into view", + ]); + assert.equal(rows().some(row => row.querySelector(".lucide-chevron-right")), false, + "an action row records a step, so it does not imply a submenu"); + // The rows that lead somewhere are the ones carrying a chevron: the rest of the runner's + // surface, and the branch that describes a step instead of pinning one. + assert.deepEqual([...menu()!.querySelectorAll("[data-recorder-panel]")].map(row => (row as HTMLElement).dataset.recorderPanel), + ["commands", "agentic"]); + assert.ok(rows().every(row => row.className.includes("hover:bg-accent") || row.parentElement?.className.includes("hover:bg-accent")), + "every action row has a visible hover state"); + assert.ok(menu()?.className.includes("cursor-pointer"), "popover buttons use a pointer cursor"); + assert.equal(rows()[0].title, 'screen.getByRole("button", "Go To Explore List").tap()'); + // The menu is placed against the window, not inside the stage: the stage clips what it holds, + // and a menu cut off at the bottom of a short panel is worse than one lying over the mirror. + const box = menu() as HTMLElement; + assert.equal(box.parentElement, dom.window.document.body, "it is a portal, so nothing can clip it"); + assert.ok(box.className.includes("fixed"), "it is placed against the window, not inside the stage"); + const width = Math.max(180, Math.min(280, dom.window.innerWidth - 16)); + assert.equal(box.style.width, `${width}px`); + // The element's own left edge in window coordinates, pushed in from the window's edge if needed. + const left = 10 + 100 / 402 * 300; + assert.equal(box.style.left, `${Math.max(8, Math.min(left, dom.window.innerWidth - width - 8))}px`); + // Below the element, since this window has room for it there. + const bottom = 20 + (400 + 44) / 874 * 650; + assert.equal(box.style.top, `${bottom + 6}px`); + assert.equal(box.style.visibility, "visible", "it is shown only once it has been measured"); + + await act(async () => { rows().find(row => row.dataset.recorderOffer === "assertVisible")!.click(); }); + await act(async () => { await Promise.resolve(); }); + assert.equal(menu(), null, "choosing a row closes the menu"); + assert.deepEqual(testRecorder.getSnapshot().steps.map((step: any) => step.kind), ["assertVisible"]); + assert.match(testRecorder.source, /await expect\(screen\.getByRole\("button", "Go To Explore List"\)\)\.toBeVisible\(\);/); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("a row that needs a value asks for it instead of emitting an empty string", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-fill-row", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const field = { + name: "Search transactions", placeholder: "Search transactions", value: "Paris", role: "textbox", + depth: 1, source: "accessibility" as const, nodeId: "field", bounds: { x: 100, y: 400, width: 200, height: 44 }, + }; + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + const done: unknown[] = []; + testRecorder.attach("udid", "ios", { + readComponents: async () => [field], screen: () => SCREEN, release: () => undefined, + perform: (action: unknown) => { done.push(action); }, + }); + testRecorder.start("udid", "ios"); + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const rows = () => [...dom.window.document.querySelectorAll("[data-recorder-offer]")] as HTMLButtonElement[]; + + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + // A field is the one element that gains a fourth row, because it is the one that takes text. + assert.deepEqual(rows().map(row => row.dataset.recorderOffer), ["tap", "fill", "assertVisible", "scrollUntilVisible"]); + await act(async () => { rows().find(row => row.dataset.recorderOffer === "fill")!.click(); }); + const input = dom.window.document.querySelector('[aria-label="Text to enter"]') as HTMLInputElement; + assert.equal(input.value, "Paris", "it starts from the field's own value"); + assert.equal(rows().length, 0, "the rows give way to the question"); + await act(async () => { input.value = "Berlin"; }); + await act(async () => { (dom.window.document.querySelector("[data-element=recorder-menu] form") as HTMLFormElement).requestSubmit(); }); + await act(async () => { await Promise.resolve(); }); + assert.match(testRecorder.source, /await screen\.getByPlaceholder\("Search transactions"\)\.fill\("Berlin"\);/); + // Typing is taken as well as recorded: the field is tapped to focus it, then the text is sent. + assert.deepEqual(done, [{ kind: "tap", point: { x: 200, y: 422 } }, { kind: "type", text: "Berlin" }]); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("a run marks the preview line by line, and marks nothing it was not told about", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-view", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const panelListeners = new Set<() => void>(); + let panelState: Record<string, unknown> = { + open: true, available: true, projectPath: "/app", projects: [], targets: [{ name: "ios", platform: "ios", bundleId: "com.app" }], + targetName: "ios", configPath: "/app/e2e.config.ts", loadingProject: false, pickingProject: false, launching: false, + appLaunched: true, fileName: "flow", saving: false, overwritePath: "", savedRevision: -1, savedPath: "", running: false, + pendingRun: false, drafts: [], draftId: "", loadingDraft: false, tests: [], deletingTest: "", error: null, + }; + /** What Run was asked for, so Play until here can be told apart from the Run test button. */ + const runs: unknown[] = []; + const panel = { + subscribe: (listener: () => void) => { panelListeners.add(listener); return () => { panelListeners.delete(listener); }; }, + getSnapshot: () => panelState, + canLaunch: true, canRun: true, canRunStored: true, dirty: true, + refreshProjects: async () => {}, chooseProject: async () => {}, pickProject: async () => {}, + launchApp: async () => {}, saveTest: async () => {}, runTest: async (options?: unknown) => { runs.push(options); }, selectTarget: () => {}, + setFileName: () => {}, cancelOverwrite: () => {}, dismissError: () => {}, + refreshDrafts: async () => {}, openDraft: async () => {}, deleteDraft: async () => {}, newRecording: async () => {}, + refreshTests: async () => {}, openTest: async () => {}, editStoredTest: () => {}, saveStoredTest: async () => {}, undoStoredTest: async () => {}, runStoredTest: async () => {}, deleteTest: async () => {}, + }; + const deviceApps = { + subscribe: () => () => undefined, + getSnapshot: () => DEVICE_SNAPSHOT, + }; + + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + await testRecorder.openMenu({ x: 150, y: 420 }); + await testRecorder.chooseOffer("tap"); + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const marks = () => [...dom.window.document.querySelectorAll("[data-testing-preview] .testing-run-mark")] + .map(mark => mark.className.replace("testing-run-mark", "").trim()); + const preview = () => (dom.window.document.querySelector("[data-testing-preview]") as HTMLElement).innerText + ?? dom.window.document.querySelector("[data-testing-preview]")!.textContent!; + + assert.equal(dom.window.document.querySelector("[aria-label=Testing] dl"), null, + "project, config, target, bundle and device metadata no longer duplicate the controls"); + const testing = dom.window.document.querySelector("[aria-label=Testing]"); + assert.ok(testing?.className.includes("cursor-pointer"), "testing panel buttons use a pointer cursor"); + // The title and file name fields are folded into Test Description: they are touched once per + // recording, at the end, while the step list beside them is touched on every tap. + assert.equal(dom.window.document.querySelector("[data-testing-title]"), null, + "the naming fields start folded away"); + const description = dom.window.document.querySelector("[data-testing-description-toggle]") as HTMLButtonElement; + assert.equal(description.getAttribute("aria-expanded"), "false"); + assert.match(description.textContent ?? "", /Test Description/); + await act(async () => { description.click(); }); + assert.ok(dom.window.document.querySelector("[data-testing-title]"), "opening it shows the title field"); + assert.ok(dom.window.document.querySelector("[data-testing-filename]"), "and the file name field"); + assert.ok(dom.window.document.querySelector("[data-testing-description]")?.contains( + dom.window.document.querySelector("[data-testing-save]")), "Save moves inside the section with them"); + // Closed, the row reads as the name the flow will be saved under, clipped at a width of its own + // so it degrades the same way wherever the pane ends. The file name is not beside it: at the + // width this pane gets, a name and a path together were two truncated strings reading as neither, + // and the path is in the File name field a click below. + const readout = dom.window.document.querySelector("[data-testing-description-title]") as HTMLElement; + assert.ok(readout, "the name is the header's readout"); + assert.ok(readout.className.includes("max-w-[24ch]"), "it has a text width of its own"); + assert.ok(readout.className.includes("truncate"), "and clips rather than wrapping"); + assert.equal(readout.title, "a recorded flow", "with the whole value as a tooltip"); + assert.equal(dom.window.document.querySelector("[data-testing-description-file]"), null, + "the file name is not duplicated in the header"); + assert.doesNotMatch(description.textContent ?? "", /tests\//, "nor is the path, under any other name"); + assert.ok(!dom.window.document.querySelector("[data-testing-title]")?.closest(".grid")?.className.includes("bg-card"), + "title and file fields have no card fill"); + assert.ok([...dom.window.document.querySelectorAll("[data-testing-step]")] + .every(step => !step.className.includes("bg-card")), "step rows have no card fill"); + // The preview and the steps are dragged apart rather than fixed at 3fr/2fr, so the group is a + // resizable one with a handle between its two panes. + const workspace = dom.window.document.querySelector("[data-testing-recorder-workspace]")!; + const panes = [...workspace.children].filter(child => child.getAttribute("data-slot") === "resizable-panel"); + assert.deepEqual(panes.map(pane => pane.id), ["testing-preview-resizable", "testing-steps-resizable"], + "the preview stays to the left of the test editor"); + assert.ok(panes[0].querySelector("[data-testing-preview-pane]")); + assert.ok(panes[1].querySelector("[data-testing-editor-pane]")); + assert.ok(workspace.querySelector("[data-slot=resizable-handle]"), "and they can be dragged apart"); + assert.match(workspace.textContent ?? "", /Preview.*Test Description.*Steps/s, + "the recorder is organized around preview, naming and steps"); + await act(async () => { description.click(); }); + const header = dom.window.document.querySelector("[aria-label=Testing] > header")!; + assert.ok(header.contains(dom.window.document.querySelector("[data-testing-controls]")), + "record and run controls share the project and launch bar"); + + // One step, one call line: line 8 of the emitted file, which is where the report will point. + const spans = testRecorder.emitted.spans; + assert.deepEqual(spans.map((span: any) => span.calls), [[8]]); + const lines = testRecorder.source.split("\n"); + assert.match(lines[7], /await screen\.getByRole\("button", "Go To Explore List"\)\.tap\(\);/); + + const step = dom.window.document.querySelector("[data-testing-step]")!; + await act(async () => { step.dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + assert.ok(dom.window.document.querySelector('[data-testing-preview-line="8"]')?.classList.contains("is-step-highlighted"), + "hovering a step highlights the line it emits"); + await act(async () => { step.dispatchEvent(new dom.window.MouseEvent("mouseout", { bubbles: true })); }); + assert.equal(dom.window.document.querySelector(".testing-preview-line.is-step-highlighted"), null, + "the emitted line returns to normal when the step is no longer hovered"); + + // The gutter is a column of its own, not a character at the front of the line: that is what keeps + // the code still when a run starts and keeps a mark level with a line that wraps. The line number + // is a third column in front of it, for the same reason and read the same way. + const line = dom.window.document.querySelector("[data-testing-preview] .testing-preview-line")!; + assert.deepEqual([...line.children].map(child => child.className.split(" ")[0]), + ["testing-preview-number", "testing-run-mark", "testing-preview-code"], + "the number, the mark and the code are siblings, in that order"); + assert.deepEqual([...dom.window.document.querySelectorAll("[data-testing-preview] .testing-preview-number")] + .slice(0, 3).map(number => number.textContent), ["1", "2", "3"], "numbered from the file's own first line"); + + // Play until here: the gutter's other occupant, on the lines a step emitted and nowhere else. + const plays = () => [...dom.window.document.querySelectorAll("[data-testing-preview] .testing-play-until")] + .map(play => play.getAttribute("data-testing-play-until")); + assert.deepEqual(plays(), ["6", "8"], + "the step's own line offers to run the flow up to it, and so does the app.open above it"); + // A play beside a comment would promise a line the runner never executes, and the gutter marks + // only the awaited calls, so the two have to be the same set of lines. + assert.deepEqual(plays().map(at => lines[Number(at) - 1].trim().startsWith("//")), [false, false], + "a comment is not somewhere a run can stop"); + assert.deepEqual(plays(), testRecorder.emitted.calls.map(String), + "exactly the lines the runner reports a step for"); + const play = dom.window.document.querySelector('[data-testing-preview-line="8"] .testing-play-until') as HTMLButtonElement; + assert.ok(line.querySelector(".testing-play-until") === null, "and it is not on the import at the top of the file"); + // The app opening is where a flow most often breaks, and it is the one call the header holds. + const opening = dom.window.document.querySelector('[data-testing-preview-line="6"] .testing-play-until') as HTMLButtonElement; + assert.match(opening.getAttribute("aria-label") ?? "", /^Run the test up to the app opening, line 6$/); + await act(async () => { opening.click(); }); + assert.deepEqual(runs, [{ untilStepId: "app:open" }], "that run opens the app and does nothing else"); + runs.length = 0; + assert.match(play.getAttribute("aria-label") ?? "", /^Run the test up to .*, line 8$/, + "the gutter's mark is decoration, but this is a control and says what it does"); + await act(async () => { play.click(); }); + assert.deepEqual(runs, [{ untilStepId: testRecorder.getSnapshot().steps[0].id }], + "clicking it runs the flow as far as that step, not the whole file"); + + await act(async () => { testRecorder.beginRun(); }); + assert.deepEqual(plays(), [], "the gutter is the run's own while it is in flight, so no play hides a spinner"); + assert.deepEqual(marks().filter(mark => mark !== ""), [], "nothing is marked until the runner says where it is"); + await act(async () => { testRecorder.applyRunProgress([{ stepIndex: 1, phase: "start" }]); }); + assert.equal(marks().filter(mark => mark === "is-running").length, 1, "one line spins: the one the runner is on"); + assert.equal(marks()[7], "is-running", "and it is the line the step emitted"); + + await act(async () => { testRecorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }] }); }); + assert.equal(marks()[7], "is-passed"); + assert.match(dom.window.document.body.textContent ?? "", /Passed/); + + await act(async () => { testRecorder.finishRun({ status: "failed", steps: [{ line: 8, status: "failed", error: "Timed out waiting for the element" }] }); }); + assert.equal(marks()[7], "is-failed", "a failure stays on the line the report blames"); + assert.match(dom.window.document.body.textContent ?? "", /Timed out waiting for the element/); + + // A run that produced no report has no line to mark, so the runner's own words stand in. + await act(async () => { testRecorder.finishRun({ status: "failed", steps: [], output: "No tests matched" }); }); + assert.deepEqual(marks().filter(mark => mark !== ""), [], "no invented marks"); + assert.match(dom.window.document.body.textContent ?? "", /No tests matched/); + + testRecorder.clearRun(); + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("the element menu survives a tree invalidation that takes the hover outline away", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-menu-anchor", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + assert.ok(dom.window.document.querySelector("[data-element=recorder-menu]")); + // The hover outline is the first thing an input, a frame or a slow read drops. The menu was asked + // for explicitly and carries its own rectangle, so it is still on the element it is about. + await act(async () => { (testRecorder as any).update({ hover: undefined }); }); + const menu = dom.window.document.querySelector("[data-element=recorder-menu]") as HTMLElement; + assert.ok(menu, "the menu does not go with the outline"); + const stays = Math.max(180, Math.min(280, dom.window.innerWidth - 16)); + assert.equal(menu.style.left, `${Math.max(8, Math.min(10 + 100 / 402 * 300, dom.window.innerWidth - stays - 8))}px`); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("a coordinate gets the tap row and is told why that is all", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-point-menu", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + + // Nowhere near the one element in the tree, so nothing can be named. + await act(async () => { await testRecorder.openMenu({ x: 10, y: 20 }); }); + const rows = [...dom.window.document.querySelectorAll("[data-recorder-offer]")] as HTMLElement[]; + assert.deepEqual(rows.map(row => row.dataset.recorderOffer), ["tap"]); + assert.equal(rows[0].textContent, "Tap"); + assert.match(rows[0].title, /screen\.tapAt\(\{ x: 10, y: 20 \}\)/); + // Three missing rows with no explanation read as a broken menu — and the note has to name what + // is still possible here, since two commands now are. + assert.match(dom.window.document.querySelector("[data-element=recorder-menu]")?.textContent ?? "", + /The coordinate can still be tapped or swiped/); + + // `screen.swipe` is about the viewport, not about anything in it, so it survives the refusal + // every other non-tap command meets on a point. + await act(async () => { dom.window.document.querySelector<HTMLElement>("[data-recorder-panel=commands]")!.click(); }); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 150)); }); + const swipeRow = dom.window.document.querySelector<HTMLElement>("[data-recorder-panel=swipe]")!; + assert.ok(swipeRow, "the viewport swipe is offered where nothing can be named"); + await act(async () => { swipeRow.dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 150)); }); + assert.deepEqual([...dom.window.document.querySelectorAll("[data-recorder-swipe]")].map(row => row.textContent), + ["Up", "Down", "Left", "Right"]); + await act(async () => { dom.window.document.querySelector<HTMLElement>("[data-recorder-swipe=down]")!.click(); }); + await act(async () => { await Promise.resolve(); }); + assert.deepEqual(testRecorder.getSnapshot().steps.map((step: any) => [step.kind, step.direction]), [["swipe", "down"]]); + assert.match(testRecorder.source, /await screen\.swipe\(\{ direction: "down" \}\);/); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("the command list is a submenu of its own, opened by pointing at its row", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-submenu", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const menu = () => dom.window.document.querySelector("[data-element=recorder-menu]") as HTMLElement; + const submenu = () => dom.window.document.querySelector("[data-element=recorder-commands]") as HTMLElement; + const rows = () => [...dom.window.document.querySelectorAll("[data-recorder-offer]")] as HTMLButtonElement[]; + const opener = () => dom.window.document.querySelector("[data-recorder-panel=commands]") as HTMLButtonElement; + + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + // The short list is what is in front of the user, and the submenu is not open until it is asked for. + assert.deepEqual(rows().map(row => row.textContent), ["Tap", "Is Visible", "Scroll into view"]); + assert.equal(submenu(), null); + + // Pointing at the row is enough; no click is needed to see what it leads to. + await act(async () => { opener().dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + assert.ok(submenu(), "hovering the row opens the submenu"); + // A panel of its own, beside the menu rather than replacing what was in it, and a portal too: + // the menu scrolls once it is taller than the window, and a scrolling box clips what hangs out. + assert.equal(submenu().parentElement, dom.window.document.body); + assert.ok(submenu().className.includes("fixed")); + // Where it lands is `popover-placement`'s arithmetic, checked there against real layouts; what + // matters here is that it is narrower than the menu, so a tight window keeps part of the menu + // behind it visible. + assert.ok(Number.parseFloat(submenu().style.width) < Number.parseFloat(menu().style.width)); + assert.ok(submenu().className.includes("bg-popover") && submenu().className.includes("shadow-lg")); + assert.deepEqual(rows().slice(0, 3).map(row => row.textContent), ["Tap", "Is Visible", "Scroll into view"], + "the menu behind it keeps its own rows"); + const commands = rows().slice(3).map(row => row.dataset.recorderOffer); + assert.ok(commands.includes("assertNotVisible") && commands.includes("doubleTap") && commands.includes("longPress")); + assert.match(submenu().textContent ?? "", /ActionsTap.*AssertionsIs Visible/); + + // Leaving the row and its panel closes it, but only after a moment: they have a gap between + // them, and crossing it must not be a way out. + await act(async () => { opener().parentElement!.dispatchEvent(new dom.window.MouseEvent("mouseleave", { bubbles: false })); }); + assert.ok(submenu(), "it is not taken away the instant the pointer leaves the row"); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 200)); }); + assert.equal(submenu(), null); + + await act(async () => { opener().dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + await act(async () => { rows().find(row => row.dataset.recorderOffer === "assertNotVisible")!.click(); }); + await act(async () => { await Promise.resolve(); }); + assert.equal(dom.window.document.querySelector("[data-element=recorder-menu]"), null, "choosing a command closes the menu"); + assert.match(testRecorder.source, /await expect\(screen\.getByRole\("button", "Go To Explore List"\)\)\.not\.toBeVisible\(\);/); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("the agentic branches cascade to the side, and record a sentence with its params", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-agentic", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + const done: unknown[] = []; + testRecorder.attach("udid", "ios", { + readComponents: async () => [button], screen: () => SCREEN, release: () => undefined, + perform: (action: unknown) => { done.push(action); }, + }); + testRecorder.start("udid", "ios"); + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const doc = dom.window.document; + const panel = (name: string) => doc.querySelector(`[data-element=${name}]`) as HTMLElement | null; + const find = (selector: string) => doc.querySelector(selector) as HTMLElement | null; + const type = async (field: HTMLInputElement, value: string) => { + const setter = Object.getOwnPropertyDescriptor(dom.window.HTMLInputElement.prototype, "value")!.set!; + await act(async () => { + setter.call(field, value); + field.dispatchEvent(new dom.window.Event("input", { bubbles: true })); + }); + }; + // Hover intent: a branch opens by being pointed at, after a beat, so a row brushed past on the + // way somewhere else does not open one. + const hover = async (node: HTMLElement) => { + await act(async () => { node.dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 150)); }); + }; + + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + const agenticRow = find("[data-recorder-panel=agentic]")!; + assert.equal(panel("recorder-agentic"), null, "a branch is not open before it is pointed at"); + await hover(agenticRow); + assert.ok(panel("recorder-agentic"), "hovering opens it; no click needed"); + // The menu it came from is still there: a branch opens beside its parent, it does not replace it. + assert.ok(panel("recorder-menu"), "the element menu stays on screen behind the branch"); + assert.ok(find("[data-recorder-panel=agentic]"), "including the row the branch hangs off"); + assert.deepEqual([...panel("recorder-agentic")!.querySelectorAll("[data-recorder-agentic]")].map(row => row.textContent), + ["Act", "Assert", "Wait for"]); + + await hover(find("[data-recorder-agentic=agentAct]")!); + const actPanel = panel("recorder-agentic-act")!; + assert.ok(actPanel, "the form cascades off Act rather than replacing the list"); + assert.ok(panel("recorder-agentic"), "which is still behind it"); + const instruction = find("[data-recorder-agentic-input=agentAct]") as HTMLInputElement; + // The element the user right-clicked writes the first draft; it is editable and nothing else. + assert.equal(instruction.value, 'tap the "Go To Explore List" button'); + await type(instruction, "sign in as the test user"); + + // Params: key and text rows, added and removed, with blank keys dropped on the way out. + await act(async () => { find("[data-recorder-param-add]")!.click(); }); + await act(async () => { find("[data-recorder-param-add]")!.click(); }); + await act(async () => { find("[data-recorder-param-add]")!.click(); }); + await type(find("[data-recorder-param-key='0']") as HTMLInputElement, "email"); + await type(find("[data-recorder-param-value='0']") as HTMLInputElement, "a@b.com"); + await type(find("[data-recorder-param-key='1']") as HTMLInputElement, "attempts"); + await type(find("[data-recorder-param-value='1']") as HTMLInputElement, "3"); + // The third row is never named, so it is not a param. + await type(find("[data-recorder-param-value='2']") as HTMLInputElement, "ignored"); + await act(async () => { find("[data-recorder-param-remove='2']")!.click(); }); + assert.equal(doc.querySelectorAll("[data-recorder-param]").length, 2, "a row can be taken back out"); + + await act(async () => { panel("recorder-agentic-act")!.querySelector("form")!.dispatchEvent(new dom.window.Event("submit", { bubbles: true, cancelable: true })); }); + await act(async () => { await Promise.resolve(); }); + assert.deepEqual(testRecorder.getSnapshot().steps.map((step: any) => [step.kind, step.instruction, step.params]), + [["agentAct", "sign in as the test user", { email: "a@b.com", attempts: "3" }]]); + assert.match(testRecorder.source, /await agent\.act\("sign in as the test user", \{ params: \{ email: "a@b\.com", attempts: 3 \} \}\);/); + assert.deepEqual(done, [], "an agent step is not taken on the device while recording"); + + // Assert cascades the same way, and its vision choice is a branch of its own. + testRecorder.clearSteps(); + await act(async () => { await testRecorder.openMenu({ x: 150, y: 420 }); }); + await hover(find("[data-recorder-panel=agentic]")!); + await hover(find("[data-recorder-agentic=agentAssert]")!); + assert.ok(panel("recorder-agentic-assert")); + await type(find("[data-recorder-agentic-input=agentAssert]") as HTMLInputElement, "the city list is showing"); + const visionRow = find("[data-recorder-panel=vision]")!; + assert.match(visionRow.textContent ?? "", /Vision: Off/, "the runner's own default, named on the row"); + await hover(visionRow); + assert.deepEqual([...panel("recorder-vision")!.querySelectorAll("[data-recorder-vision]")].map(row => row.textContent), + ["Off", "On", "Screenshot only"]); + await act(async () => { find("[data-recorder-vision=only]")!.click(); }); + assert.match(find("[data-recorder-panel=vision]")!.textContent ?? "", /Vision: Screenshot only/); + await act(async () => { panel("recorder-agentic-assert")!.querySelector("form")!.dispatchEvent(new dom.window.Event("submit", { bubbles: true, cancelable: true })); }); + await act(async () => { await Promise.resolve(); }); + + assert.equal(panel("recorder-menu"), null, "recording the step closes the whole menu"); + assert.deepEqual(testRecorder.getSnapshot().steps.map((step: any) => [step.kind, step.vision]), [["agentAssert", "only"]]); + assert.match(testRecorder.source, /await agent\.assert\("the city list is showing", \{ vision: "only" \}\);/); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +/** A `TestingPanel` stub for the view: enough snapshot for the controls, no server behind it. */ +function panelStub(overrides: Record<string, unknown> = {}) { + const listeners = new Set<() => void>(); + const state: Record<string, unknown> = { + open: true, available: true, projectPath: "/app", projects: [], targets: [{ name: "ios", platform: "ios", bundleId: "com.app" }], + targetName: "ios", configPath: "/app/e2e.config.ts", loadingProject: false, pickingProject: false, launching: false, + appLaunched: true, fileName: "flow", saving: false, overwritePath: "", savedRevision: -1, savedPath: "", running: false, + pendingRun: false, agent: null, drafts: [], draftId: "", loadingDraft: false, tests: [], deletingTest: "", error: null, + ...overrides, + }; + return { + subscribe: (listener: () => void) => { listeners.add(listener); return () => { listeners.delete(listener); }; }, + getSnapshot: () => state, + canLaunch: true, canRun: true, canRunStored: true, dirty: true, + refreshProjects: async () => {}, chooseProject: async () => {}, pickProject: async () => {}, + launchApp: async () => {}, saveTest: async () => {}, runTest: async () => {}, selectTarget: () => {}, + setFileName: () => {}, cancelOverwrite: () => {}, dismissError: () => {}, + refreshDrafts: async () => {}, openDraft: async () => {}, deleteDraft: async () => {}, newRecording: async () => {}, + refreshTests: async () => {}, openTest: async () => {}, editStoredTest: () => {}, saveStoredTest: async () => {}, undoStoredTest: async () => {}, runStoredTest: async () => {}, deleteTest: async () => {}, + }; +} + +test("a step's commands open beside it, and Clear asks before it throws the flow away", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-step-menu", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const panel = panelStub(); + const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + await testRecorder.openMenu({ x: 150, y: 420 }); + await testRecorder.chooseOffer("tap"); + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const query = (selector: string) => dom.window.document.querySelector(selector) as HTMLElement | null; + + // The step's own menu, revealed by pointing at the row it belongs to. + const trigger = query("[data-testing-step-menu]")!; + await act(async () => { trigger.dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + const commandsRow = query("[data-testing-step-commands]"); + assert.ok(commandsRow, "the menu offers the step another command"); + assert.ok(query("[data-testing-step-duplicate]"), "beside Duplicate"); + assert.equal(query("[data-testing-step-commands-back]"), null, + "the list is a branch now, so there is nothing to go back from"); + + // Pointing at that row opens the list beside the menu, not over it. + await act(async () => { commandsRow!.dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + await act(async () => { await new Promise(resolve => setTimeout(resolve, 150)); }); + const branch = query("[data-element=testing-step-commands]"); + assert.ok(branch, "hovering the row opens the command list"); + assert.equal(branch!.parentElement, dom.window.document.body, "as a portal, so nothing can clip it"); + assert.ok(query("[data-testing-step-duplicate]"), "and the menu it came from is still on screen"); + assert.ok(branch!.textContent?.includes("Actions"), "the list is the runner's own command vocabulary"); + // That it opens on the *left* is `placeBeside`'s arithmetic, checked in + // `packages/ui/test/popover-placement.test.ts` against real layouts: every rectangle in JSDOM is + // zero, so there is no left here to open on. + + // Clear is the icon alone, tinted, and it asks before it acts. + const clear = query("[data-testing-clear]")!; + assert.equal(clear.textContent?.trim(), "", "no text label beside the icon"); + assert.equal(clear.getAttribute("aria-label"), "Clear test"); + assert.ok(clear.className.includes("text-destructive"), "and it is tinted as the destructive thing it is"); + await act(async () => { clear.click(); }); + const confirm = query("[data-testing-clear-confirm]"); + assert.ok(confirm, "clicking it asks first"); + assert.equal(testRecorder.getSnapshot().steps.length, 1, "and nothing has been cleared yet"); + await act(async () => { confirm!.click(); }); + assert.equal(testRecorder.getSnapshot().steps.length, 0, "answering it clears the flow"); + + testRecorder.detach("udid", "ios"); +}); + +test("Record says the mirror is not ready yet, until selection actually is", async t => { + const { module: { TestRecorderOverlay, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestRecorderOverlay } from "./packages/ui/src/components/test-recorder-overlay.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "recorder-preparing", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + // The real wait: attaching to the app and projecting its tree takes a second or two, and until + // it lands nothing on the mirror can be named. + let release: (components: unknown[]) => void = () => {}; + const tree = new Promise<unknown[]>(resolve => { release = resolve; }); + const annotations = { subscribe: () => () => undefined, getSnapshot: () => ANNOTATIONS_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: () => tree, screen: () => SCREEN, release: () => undefined }); + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestRecorderOverlay, { store: annotations })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const overlay = () => dom.window.document.querySelector("[data-element=recorder-preparing]") as HTMLElement | null; + + assert.equal(overlay(), null, "nothing is over the mirror until Record is pressed"); + await act(async () => { testRecorder.start("udid", "ios"); }); + assert.ok(overlay(), "Record puts it there at once, rather than after the first dead click"); + assert.match(overlay()!.textContent ?? "", /preparing\.\.\./); + assert.ok(overlay()!.className.includes("pointer-events-none"), + "the mirror still drives the app while the tree is being read"); + + await act(async () => { release([button]); await tree; }); + assert.equal(overlay(), null, "it goes when an element can actually be selected"); + assert.equal(testRecorder.getSnapshot().preparing, false); + + testRecorder.stop(); + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); + +test("the list keeps what is not stored above what the project holds, and asks before deleting either", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-list-groups", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const deleted: string[] = []; + const previewed: string[] = []; + const panel = { + ...panelStub({ + drafts: [{ id: "draft-1", projectPath: "/app", title: "the explore screen", fileName: "", origin: "user", stepCount: 3, createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z" }], + tests: [{ relativePath: "tests/flow.e2e.ts", name: "flow.e2e.ts", updatedAt: "2026-01-01T00:00:00.000Z", sizeBytes: 420 }], + }), + deleteTest: async (path: string) => { deleted.push(path); }, + openTest: async (path: string) => { previewed.push(path); }, + }; + const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT }; + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const query = (selector: string) => dom.window.document.querySelector(selector) as HTMLElement | null; + + await act(async () => { query("[data-testing-drafts]")!.click(); }); + const groups = [...dom.window.document.querySelectorAll("[data-testing-group]")] as HTMLElement[]; + assert.deepEqual(groups.map(group => group.dataset.testingGroup), ["unstored", "stored"], + "what is being worked on is above the archive"); + assert.match(groups[0].textContent ?? "", /Not stored/); + // Expanded by default: the unsaved recordings are the ones at risk of being lost. + assert.equal(groups[0].getAttribute("aria-expanded"), "true"); + assert.ok(query("[data-testing-draft-option=draft-1]")); + // Collapsed by default: the project's own tests are there to be found, not to be read every time. + assert.equal(groups[1].getAttribute("aria-expanded"), "false"); + assert.equal(query("[data-testing-test-delete]"), null); + + await act(async () => { groups[1].click(); }); + assert.match(query('[data-testing-test="tests/flow.e2e.ts"]')?.textContent ?? "", /flow\.e2e\.ts/, + "a saved test is listed here rather than leaving the list"); + await act(async () => { query('[data-testing-test-option="tests/flow.e2e.ts"]')!.click(); }); + assert.deepEqual(previewed, ["tests/flow.e2e.ts"], "the project row opens a preview"); + await act(async () => { query("[data-testing-drafts]")!.click(); }); + + // Neither delete happens on one click of a trash icon sitting beside the row the user wanted. + // The question is a popover beside the icon, so the row it is about stays readable under it. + await act(async () => { query("[data-testing-test-delete]")!.click(); }); + assert.deepEqual(deleted, [], "nothing is deleted while the question is still up"); + const question = query("[role=alertdialog]"); + assert.match(question?.textContent ?? "", + /Delete tests\/flow\.e2e\.ts from the project\? The file is removed and cannot be brought back/); + assert.equal(question?.parentElement?.closest("[role=alertdialog]"), null, "it is a popover, not the row"); + assert.ok(query('[data-testing-test="tests/flow.e2e.ts"]'), "and the row it is about is still listed"); + await act(async () => { query("[data-confirm-delete]")!.click(); }); + assert.deepEqual(deleted, ["tests/flow.e2e.ts"]); + assert.equal(query("[role=alertdialog]"), null, "answering it closes the question"); + + testRecorder.clearSteps(); +}); + +test("a selected project test offers temporary step edits and per-line runs", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-project-preview", + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + const edits: string[] = []; + let saves = 0; + const panel = { ...panelStub({ + tests: [{ relativePath: "tests/flow.e2e.ts", name: "flow.e2e.ts", updatedAt: "2026-01-01T00:00:00.000Z", sizeBytes: 30 }], + selectedTestPath: "tests/flow.e2e.ts", + selectedTestSource: 'test("stored", async ({ app, screen }) => {\n await app.open();\n await screen.getByText("Done").tap();\n});', + loadingTest: false, selectedTestStaged: true, + }), editStoredTest: (source: string) => { edits.push(source); }, saveStoredTest: async () => { saves += 1; } }; + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps: { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT } })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const query = (selector: string) => dom.window.document.querySelector(selector); + assert.match(query("[data-testing-stored-preview]")?.textContent ?? "", /stored/); + assert.match(query("[data-testing-stored-steps]")?.textContent ?? "", /screen\.getByText\("Done"\)\.tap\(\)/); + assert.equal(query("[data-testing-stored-step]")?.getAttribute("data-testing-stored-step"), "3"); + assert.ok(query('[data-testing-play-until="3"]'), "a saved call offers play until here"); + assert.doesNotMatch(query("[data-testing-stored-step]")?.textContent ?? "", /:3/, "the source line is not repeated in the step row"); + assert.ok(query("[data-testing-stored-save]")); + assert.ok(query("[data-testing-stored-undo]")); + await act(async () => { (query("[data-testing-source-edit]") as HTMLElement).click(); }); + assert.ok(query("[data-testing-source-input]"), "the whole source opens for editing"); + assert.match(query("[data-testing-source-edit]")?.parentElement?.textContent ?? "", /^PreviewEdit/); + assert.equal(query("[data-testing-source-edit]")?.getAttribute("aria-pressed"), "true"); + assert.ok(query(".testing-source-editor-code")?.classList.contains("bg-muted/40"), "Edit shares Preview's surface"); + assert.ok(query(".testing-source-editor-code")?.classList.contains("dark:bg-black/15")); + assert.equal(query("[data-testing-source-line-number]")?.textContent, "1"); + assert.ok(query("[data-testing-source-highlight] .text-emerald-600"), "strings retain preview syntax colors"); + const suggestionInput = query("[data-testing-source-input]") as HTMLTextAreaElement; + const originalSource = suggestionInput.value; + const setter = Object.getOwnPropertyDescriptor(dom.window.HTMLTextAreaElement.prototype, "value")!.set!; + await act(async () => { + setter.call(suggestionInput, `${suggestionInput.value}\nawait screen.getByT`); + suggestionInput.setSelectionRange(suggestionInput.value.length, suggestionInput.value.length); + suggestionInput.dispatchEvent(new dom.window.Event("input", { bubbles: true })); + }); + assert.deepEqual([...dom.window.document.querySelectorAll("[data-testing-source-suggestion]")] + .map(element => element.getAttribute("data-testing-source-suggestion")), ["getByTestId", "getByText"]); + await act(async () => { suggestionInput.dispatchEvent(new dom.window.KeyboardEvent("keydown", { key: "ArrowDown", bubbles: true })); }); + await act(async () => { suggestionInput.dispatchEvent(new dom.window.KeyboardEvent("keydown", { key: "Enter", bubbles: true })); }); + assert.match(suggestionInput.value, /screen\.getByText\(""\)$/); + assert.equal(query("[data-testing-source-suggestions]"), null); + for (const [typed, expected] of [ + ["await exp", "expect visible"], + ['await expect(screen.getByText("Done")).toBeVisible(', "timeout"], + ['await agent.assert("The list is visible", ', "vision: true"], + ]) { + await act(async () => { + setter.call(suggestionInput, `${originalSource}\n${typed}`); + suggestionInput.setSelectionRange(suggestionInput.value.length, suggestionInput.value.length); + suggestionInput.dispatchEvent(new dom.window.Event("input", { bubbles: true })); + }); + assert.ok(query(`[data-testing-source-suggestion="${expected}"]`), `${typed} offers ${expected}`); + } + await act(async () => { (query("[data-testing-source-cancel]") as HTMLElement).click(); }); + assert.equal(query("[data-testing-source-input]"), null, "Cancel restores the preview"); + await act(async () => { (query("[data-testing-source-edit]") as HTMLElement).click(); }); + const input = query("[data-testing-source-input]") as HTMLTextAreaElement; + await act(async () => { setter.call(input, input.value.replace("Done", "Saved")); input.dispatchEvent(new dom.window.Event("input", { bubbles: true })); }); + assert.match(query("[data-testing-source-highlight]")?.textContent ?? "", /Saved/); + await act(async () => { (query("[data-testing-source-save]") as HTMLElement).click(); }); + assert.match(edits[0] ?? "", /getByText\("Saved"\)/); + assert.equal(saves, 1, "Save writes the edited project test"); + assert.equal(query("[data-testing-stored-step]")?.getAttribute("draggable"), "true"); + assert.ok(query("[data-testing-stored-remove]"), "project test steps can be removed"); + await act(async () => { (query("[data-testing-stored-step-menu]") as HTMLElement).click(); }); + assert.ok(query("[data-testing-stored-commands]"), "Change command is a menu branch"); + assert.equal(query("[data-testing-stored-command]"), null, "commands are hidden until the branch opens"); + await act(async () => { (query("[data-testing-stored-commands]") as HTMLElement).click(); }); + assert.ok(query("[data-testing-stored-commands-panel]"), "commands open in a separate popover"); + await act(async () => { (query('[data-testing-stored-command="assertVisible"]') as HTMLElement).click(); }); + assert.match(edits[1] ?? "", /toBeVisible\(\)/); + await act(async () => { (query("[data-testing-stored-step-menu]") as HTMLElement).click(); }); + await act(async () => { (query("[data-testing-stored-duplicate]") as HTMLElement).click(); }); + assert.equal(edits.length, 3, "duplicate edits the working copy"); + await act(async () => { (query("[data-testing-stored-remove]") as HTMLElement).click(); }); + assert.equal(edits.length, 4, "remove edits the working copy"); + assert.doesNotMatch(edits[3] ?? "", /getByText\("Done"\)/); + assert.equal(query("[data-testing-editor-pane]"), null); + assert.equal(query("[data-testing-run]")?.hasAttribute("disabled"), false); + assert.ok(query("[data-testing-new-from-preview]")); + testRecorder.clearSteps(); +}); + +test("editing a recording makes its preview the source of truth", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-recording-source-edit", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + testRecorder.restore([{ id: "step-1", label: "Tap Done", kind: "tap", selector: { kind: "text", text: "Done" } }]); + let saved = ""; + const panel = { ...panelStub(), saveTest: async () => { saved = testRecorder.source; } }; + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps: { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT } })); }); + onTeardown(async () => { await act(async () => { root.unmount(); }); testRecorder.clearSteps(); }); + const query = (selector: string) => dom.window.document.querySelector(selector); + await act(async () => { (query("[data-testing-source-edit]") as HTMLElement).click(); }); + const input = query("[data-testing-source-input]") as HTMLTextAreaElement; + const value = input.value.replace('getByText("Done")', 'getByText("Saved")'); + const setter = Object.getOwnPropertyDescriptor(dom.window.HTMLTextAreaElement.prototype, "value")!.set!; + await act(async () => { setter.call(input, value); input.dispatchEvent(new dom.window.Event("input", { bubbles: true })); }); + await act(async () => { (query("[data-testing-source-save]") as HTMLElement).click(); }); + assert.equal(saved, value); + assert.match(query("[data-testing-source-steps]")?.textContent ?? "", /getByText\("Saved"\)/); + assert.equal(query("[data-testing-source-step]")?.getAttribute("draggable"), "true"); + assert.ok(query("[data-testing-source-remove]")); + await act(async () => { (query("[data-testing-source-step-menu]") as HTMLElement).click(); }); + assert.ok(query("[data-testing-source-commands]")); + await act(async () => { (query("[data-testing-source-duplicate]") as HTMLElement).click(); }); + assert.equal((testRecorder.source.match(/getByText\("Saved"\)\.tap/g) ?? []).length, 2); + assert.equal(dom.window.document.querySelectorAll("[data-testing-source-step]").length, 2); + await act(async () => { (query("[data-testing-source-remove]") as HTMLElement).click(); }); + assert.equal(dom.window.document.querySelectorAll("[data-testing-source-step]").length, 1); + assert.ok(query("[data-testing-new-from-source]")); +}); + +test("moving from one step to the next hands the highlight over without dimming the first", async t => { + const { module: { TestingView, testRecorder }, dom, onTeardown } = await renderEnvironment<any>(t, { + exports: 'export { TestingView } from "./packages/ui/src/components/testing-view.tsx";\n' + + 'export { testRecorder } from "./packages/ui/src/test-recorder.ts";', + slug: "testing-step-hover", + extraGlobals: { crypto: { randomUUID: () => `id-${Math.random()}` } }, + }); + const { act, createElement } = await import("react"); + const { createRoot } = await import("react-dom/client"); + + const panel = panelStub(); + const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => DEVICE_SNAPSHOT }; + testRecorder.attach("udid", "ios", { readComponents: async () => [button], screen: () => SCREEN, release: () => undefined }); + testRecorder.start("udid", "ios"); + await testRecorder.openMenu({ x: 150, y: 420 }); + await testRecorder.chooseOffer("tap"); + await testRecorder.openMenu({ x: 150, y: 420 }); + await testRecorder.chooseOffer("assertVisible"); + + const root = createRoot(dom.window.document.getElementById("root")!); + await act(async () => { root.render(createElement(TestingView, { panel, deviceApps })); }); + onTeardown(async () => { + await act(async () => { root.unmount(); }); + await new Promise(resolve => dom.window.requestAnimationFrame(resolve)); + }); + const steps = () => [...dom.window.document.querySelectorAll("[data-testing-step]")] as HTMLElement[]; + const highlighted = () => [...dom.window.document.querySelectorAll(".testing-preview-line.is-step-highlighted")] + .map(line => (line as HTMLElement).dataset.testingPreviewLine); + const spans = testRecorder.emitted.spans; + assert.equal(steps().length, 2); + + await act(async () => { steps()[0].dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); }); + assert.deepEqual(highlighted(), spans[0].calls.map(String), "pointing at a step tints the lines it emits"); + + // Leaving the row takes the highlight away at once. What made leaving a step *re-light* its + // lines was the preview's own arrival animation being cancelled while they were highlighted and + // restarted when they no longer were — see `.is-step-highlighted` in `style.css`, which paints + // the highlight as a layer now so there is no animation to restart. + await act(async () => { steps()[0].dispatchEvent(new dom.window.MouseEvent("mouseout", { bubbles: true })); }); + assert.deepEqual(highlighted(), [], "the highlight goes with the pointer"); + + // And a row that has already been left cannot take it from the row the pointer is on now: the + // browser dispatches the leave of one and the enter of the next together. + await act(async () => { + steps()[1].dispatchEvent(new dom.window.MouseEvent("mouseover", { bubbles: true })); + steps()[0].dispatchEvent(new dom.window.MouseEvent("mouseout", { bubbles: true })); + }); + assert.deepEqual(highlighted(), spans[1].calls.map(String), "the second step's lines are the ones tinted"); + + // The flash itself was CSS, and JSDOM applies none: the highlight suppressed the arrival + // animation that `is-new` holds on the lines a step added, so letting it back on mouse-leave + // restarted it — the lines lit up again and faded over 1400ms. The highlight is a layer now, so + // there is nothing to cancel and nothing to restart. + const css = await readFile(new URL("../../ui/src/style.css", import.meta.url), "utf-8"); + const highlightRule = css.slice(css.indexOf(".testing-preview-line.is-step-highlighted")); + assert.match(highlightRule.slice(0, 64), /is-step-highlighted::before/, + "the highlight is painted as its own layer"); + assert.doesNotMatch(highlightRule.slice(0, highlightRule.indexOf("}")), /animation/, + "and it never touches the arrival animation, which is what restarted on mouse-leave"); + + testRecorder.clearSteps(); + testRecorder.detach("udid", "ios"); +}); diff --git a/packages/core/test/testing-projects.test.ts b/packages/core/test/testing-projects.test.ts new file mode 100644 index 00000000..1a8fdc62 --- /dev/null +++ b/packages/core/test/testing-projects.test.ts @@ -0,0 +1,253 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { deleteTestFile, discardStagedTestFile, findConfig, inspectProject, listTestFiles, parseTargets, previewTestFile, readTestFile, resolveTestRunPaths, saveStagedTestFile, stageTestFile, TestingProjectChoices, writeTestFile } from "../src/testing-projects.ts"; + +const CONFIG = `import { mobile } from "@e2e-dev/mobile"; +import type { E2EConfig } from "e2e"; + +export default { + agents: { default: { model: chatgpt("gpt-6-luna") } }, + targets: [ + { + name: "web", + engine: playwright({ browser: "chromium" }), + }, + { + name: "ios", + engine: mobile({ platform: "ios" }), + app: { bundleId: "com.anonymous.expoapp57" }, + }, + ], + workers: 1, +} satisfies E2EConfig; +`; + +test("the config read names each target and attributes its bundle id", () => { + const targets = parseTargets(CONFIG); + assert.deepEqual(targets, [ + { name: "web" }, + { name: "ios", platform: "ios", bundleId: "com.anonymous.expoapp57" }, + ]); +}); + +test("a config with no targets reads as none rather than throwing", () => { + assert.deepEqual(parseTargets("export default { workers: 1 };"), []); +}); + +test("a project is the folder holding an e2e config, and one without is refused", async () => { + const root = await mkdtemp(join(tmpdir(), "testing-projects-")); + try { + const project = join(root, "app"); + const empty = join(root, "empty"); + await mkdir(project); + await mkdir(empty); + await writeFile(join(project, "e2e.config.ts"), CONFIG, "utf-8"); + + assert.equal(await findConfig(project), join(project, "e2e.config.ts")); + assert.equal(await findConfig(empty), undefined); + + const inspected = await inspectProject(project); + assert.equal(inspected.name, "app"); + assert.equal(inspected.targets.at(-1)?.bundleId, "com.anonymous.expoapp57"); + + // A folder the recorder could not write a test for never reaches the panel's list. + const choices = new TestingProjectChoices(join(root, "state"), async () => empty); + await assert.rejects(choices.record(empty), /no e2e.config.ts/); + await choices.record(project); + assert.deepEqual((await choices.list()).map(entry => entry.path), [project]); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + + +async function project(): Promise<{ root: string; path: string }> { + const root = await mkdtemp(join(tmpdir(), "testing-write-")); + const path = join(root, "app"); + await mkdir(path); + await writeFile(join(path, "e2e.config.ts"), CONFIG, "utf-8"); + return { root, path }; +} + +const SOURCE = 'test("x", async () => {});\n'; + +test("a recorded test is written into the project's tests folder", async () => { + const { root, path } = await project(); + try { + const result = await writeTestFile(path, "tests/tab-navigation.e2e.ts", SOURCE); + assert.equal(result.written, true); + assert.equal(result.existed, false); + assert.equal(result.path, join(path, "tests", "tab-navigation.e2e.ts")); + assert.equal(await readFile(result.path, "utf-8"), SOURCE); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("an existing file is reported back untouched until the user says to replace it", async () => { + const { root, path } = await project(); + try { + await writeTestFile(path, "tests/flow.e2e.ts", "first\n"); + const refused = await writeTestFile(path, "tests/flow.e2e.ts", SOURCE); + assert.equal(refused.written, false); + assert.equal(refused.existed, true); + assert.equal(await readFile(refused.path, "utf-8"), "first\n"); + + const replaced = await writeTestFile(path, "tests/flow.e2e.ts", SOURCE, true); + assert.equal(replaced.written, true); + assert.equal(await readFile(replaced.path, "utf-8"), SOURCE); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a path that leaves the tests folder is refused", async () => { + const { root, path } = await project(); + try { + for (const bad of ["../escaped.e2e.ts", "tests/../../escaped.e2e.ts", "/etc/passwd", "src/flow.e2e.ts", "tests/flow.ts", "tests/"]) { + await assert.rejects(writeTestFile(path, bad, SOURCE), /tests\/|file name|Name the test file|\.e2e\.ts/, `${bad} must be refused`); + } + // Nothing escaped into the project root or above it. + assert.equal(await readFile(join(root, "escaped.e2e.ts"), "utf-8").catch(() => undefined), undefined); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a run accepts only an existing test and the selected project's config", async () => { + const { root, path } = await project(); + try { + await writeTestFile(path, "tests/flow.e2e.ts", SOURCE); + assert.deepEqual(await resolveTestRunPaths(path, "tests/flow.e2e.ts", join(path, "e2e.config.ts")), { + projectPath: path, + relativePath: "tests/flow.e2e.ts", + configPath: join(path, "e2e.config.ts"), + }); + await assert.rejects(resolveTestRunPaths(path, "../outside.e2e.ts"), /tests\//); + await assert.rejects(resolveTestRunPaths(path, "tests/missing.e2e.ts"), /no longer there/); + await writeFile(join(path, "other.config.ts"), "throw new Error('ran arbitrary code')\n"); + await assert.rejects(resolveTestRunPaths(path, "tests/flow.e2e.ts", join(path, "other.config.ts")), /own e2e config/); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a symlinked tests folder is refused rather than followed", async () => { + const { root, path } = await project(); + try { + const outside = join(root, "outside"); + await mkdir(outside); + await symlink(outside, join(path, "tests")); + await assert.rejects(writeTestFile(path, "tests/flow.e2e.ts", SOURCE), /symbolic link/); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a folder that is not an e2e project cannot be written to", async () => { + const { root, path } = await project(); + try { + const other = join(root, "not-a-project"); + await mkdir(other); + await assert.rejects(writeTestFile(other, "tests/flow.e2e.ts", SOURCE), /no e2e.config.ts/); + assert.ok(path); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("the project's own tests are listed, newest first, and nothing else is", async () => { + const { root, path } = await project(); + try { + // Nothing saved yet is an empty list, not an error: there is no tests/ folder at all. + assert.deepEqual(await listTestFiles(path), []); + + await writeTestFile(path, "tests/first.e2e.ts", SOURCE); + await writeTestFile(path, "tests/checkout/pay.e2e.ts", SOURCE); + // Not a recorded test, and not listed: the panel's list is of tests, not of the folder. + await writeFile(join(path, "tests", "helpers.ts"), SOURCE, "utf-8"); + // Written last, so it is first: the list is ordered by when each file was written. + await writeFile(join(path, "tests", "latest.e2e.ts"), SOURCE, "utf-8"); + + const listed = await listTestFiles(path); + assert.deepEqual([...listed].sort((left, right) => left.relativePath.localeCompare(right.relativePath)) + .map(entry => entry.relativePath), + ["tests/checkout/pay.e2e.ts", "tests/first.e2e.ts", "tests/latest.e2e.ts"]); + assert.deepEqual(listed.map(entry => entry.updatedAt), [...listed.map(entry => entry.updatedAt)].sort().reverse(), + "newest first"); + const first = listed.find(entry => entry.name === "first.e2e.ts")!; + assert.equal(first.sizeBytes, Buffer.byteLength(SOURCE, "utf-8")); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a project test can be previewed without reading outside tests", async () => { + const { root, path } = await project(); + try { + await writeTestFile(path, "tests/flow.e2e.ts", SOURCE); + assert.equal(await readTestFile(path, "tests/flow.e2e.ts"), SOURCE); + await assert.rejects(readTestFile(path, "tests/missing.e2e.ts"), /no longer there/); + await assert.rejects(readTestFile(path, "tests/../../outside.e2e.ts"), /file name|tests\//); + await writeFile(join(path, "tests", "oversized.e2e.ts"), "x".repeat(256 * 1024 + 1)); + await assert.rejects(readTestFile(path, "tests/oversized.e2e.ts"), /too large/); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("saved test edits stay temporary until Save, and Undo restores the project file", async () => { + const { root, path } = await project(); + try { + const relativePath = "tests/flow.e2e.ts"; + await writeTestFile(path, relativePath, SOURCE); + const opened = await previewTestFile(path, relativePath); + await stageTestFile(path, relativePath, opened.originalHash, "edited\n"); + assert.equal(await readTestFile(path, relativePath), SOURCE, "staging never writes the project file"); + assert.deepEqual(await previewTestFile(path, relativePath), { source: "edited\n", originalHash: opened.originalHash, staged: true }); + await discardStagedTestFile(path, relativePath); + assert.equal((await previewTestFile(path, relativePath)).source, SOURCE); + + await stageTestFile(path, relativePath, opened.originalHash, "saved\n"); + await saveStagedTestFile(path, relativePath); + assert.equal(await readTestFile(path, relativePath), "saved\n"); + assert.equal((await previewTestFile(path, relativePath)).staged, false); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("Save refuses to replace a project test changed outside the panel", async () => { + const { root, path } = await project(); + try { + const relativePath = "tests/flow.e2e.ts"; + await writeTestFile(path, relativePath, SOURCE); + const opened = await previewTestFile(path, relativePath); + await stageTestFile(path, relativePath, opened.originalHash, "panel edit\n"); + await writeTestFile(path, relativePath, "external edit\n", true); + await assert.rejects(saveStagedTestFile(path, relativePath), /changed on disk/); + assert.equal(await readTestFile(path, relativePath), "external edit\n"); + await discardStagedTestFile(path, relativePath); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("a symlink under tests is skipped rather than followed out of the project", async () => { + const { root, path } = await project(); + try { + await writeTestFile(path, "tests/flow.e2e.ts", SOURCE); + const outside = join(root, "outside"); + await mkdir(outside); + await writeFile(join(outside, "elsewhere.e2e.ts"), SOURCE, "utf-8"); + await symlink(outside, join(path, "tests", "linked")); + await symlink(join(outside, "elsewhere.e2e.ts"), join(path, "tests", "linked.e2e.ts")); + + assert.deepEqual((await listTestFiles(path)).map(entry => entry.relativePath), ["tests/flow.e2e.ts"]); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test("one of the project's tests can be deleted, and only from inside tests/", async () => { + const { root, path } = await project(); + try { + await writeTestFile(path, "tests/flow.e2e.ts", SOURCE); + await writeFile(join(root, "outside.e2e.ts"), SOURCE, "utf-8"); + + const removed = await deleteTestFile(path, "tests/flow.e2e.ts"); + assert.equal(removed.removed, true); + assert.equal(removed.path, join(path, "tests", "flow.e2e.ts")); + assert.deepEqual(await listTestFiles(path), []); + + // A file that is already gone is not an error: the list the user clicked may be out of date. + assert.equal((await deleteTestFile(path, "tests/flow.e2e.ts")).removed, false); + + // The same boundary the write goes through, so a delete cannot climb out of the folder. + for (const bad of ["../outside.e2e.ts", "tests/../../outside.e2e.ts", "src/flow.e2e.ts", "tests/flow.ts"]) { + await assert.rejects(deleteTestFile(path, bad), /tests\/|file name|\.e2e\.ts/, `${bad} must be refused`); + } + assert.ok(await readFile(join(root, "outside.e2e.ts"), "utf-8")); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/packages/core/test/testing-run.test.ts b/packages/core/test/testing-run.test.ts new file mode 100644 index 00000000..699d6bbe --- /dev/null +++ b/packages/core/test/testing-run.test.ts @@ -0,0 +1,310 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { EventEmitter } from "node:events"; +import { Readable } from "node:stream"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { existsSync } from "node:fs"; +import { readFileSync } from "node:fs"; +import { parseProgressLine, parseReport, progressConfigSource, reportDuration, reportSteps, reportStatus, runRecordedTest, runnerBin, runnerEnv, untilTestPath } from "../src/testing-run.ts"; + +const FILE = "tests/a-recorded-flow.e2e.ts"; + +/** A report as the runner's `json` reporter writes one; only the fields a line mark depends on. */ +function report(steps: { line: number; status: string; api?: string; error?: unknown }[], status = "passed") { + return { + schemaVersion: "report-1", + run: { + status, + results: [{ + testId: `${FILE}::a-recorded-flow`, + attempts: [{ + index: 0, status, + steps: steps.map((step, index) => ({ + id: `a:${index}`, index, kind: "locator", api: step.api ?? "locator.tap", + label: 'getByRole("tab", name: "Explore")', status: step.status, durationMs: 12, + source: { file: FILE, line: step.line, column: 10 }, + ...(step.error ? { error: step.error } : {}), + })), + }], + }], + }, + }; +} + +test("every reported step lands on the line it came from", () => { + const steps = reportSteps(report([{ line: 8, status: "passed" }, { line: 9, status: "failed", error: { message: "Timed out\nat tests" } }], "failed"), FILE); + assert.deepEqual(steps.map(step => [step.line, step.status]), [[8, "passed"], [9, "failed"]]); + assert.equal(steps[1].error, "Timed out", "one line of the failure is what a mark can show"); + assert.equal(reportStatus(report([], "failed"), 1), "failed"); +}); + +test("a step with no line of its own is not attributed to one", () => { + // `app.open` is a real step the runner reports and has no source line; marking line 0 would put + // a checkmark on the import at the top of the file. + const raw = report([{ line: 8, status: "passed" }]); + raw.run.results[0].attempts[0].steps.unshift({ + id: "a:open", index: 0, kind: "app", api: "app.open", label: "", status: "passed", durationMs: 90, + source: { file: "", line: 0, column: 0 }, + } as never); + assert.deepEqual(reportSteps(raw, FILE).map(step => step.line), [8]); +}); + +test("a retried test is read from the attempt that decided it", () => { + const raw = report([{ line: 8, status: "failed" }], "passed"); + raw.run.results[0].attempts.push({ + index: 1, status: "passed", + steps: [{ id: "b:1", index: 1, kind: "locator", api: "locator.tap", label: "", status: "passed", durationMs: 8, source: { file: FILE, line: 8, column: 10 } }], + } as never); + assert.deepEqual(reportSteps(raw, FILE).map(step => step.status), ["passed"]); +}); + +test("the report is found even when something else wrote to the same stream", () => { + const document = JSON.stringify(report([{ line: 8, status: "passed" }])); + assert.notEqual(parseReport(`npm warn exec something\n${document}\n`), undefined); + assert.equal(parseReport("no report here"), undefined); +}); + +async function project(): Promise<string> { + const root = await mkdtemp(join(tmpdir(), "testing-run-")); + await mkdir(join(root, "node_modules/.bin"), { recursive: true }); + await writeFile(join(root, "node_modules/.bin/e2e"), "#!/bin/sh\n"); + return root; +} + +/** A spawn that answers with one canned stdout, and records how it was called. */ +function spawner(stdout: string, exitCode = 0, seen: { command?: string; args?: readonly string[]; cwd?: string } = {}) { + return ((command: string, args: readonly string[], options: { cwd?: string }) => { + seen.command = command; seen.args = args; seen.cwd = options.cwd; + const child = new EventEmitter() as EventEmitter & { stdout: Readable; stderr: Readable; kill: () => void }; + child.stdout = Readable.from([stdout]); + child.stderr = Readable.from([""]); + child.kill = () => undefined; + queueMicrotask(() => setTimeout(() => child.emit("close", exitCode), 0)); + return child; + }) as never; +} + +test("the project's own runner is spawned, never a downloaded one", async () => { + const root = await project(); + const seen: { command?: string; args?: readonly string[]; cwd?: string } = {}; + const result = await runRecordedTest({ projectPath: root, relativePath: FILE }, + { spawner: spawner(JSON.stringify(report([{ line: 8, status: "passed" }])), 0, seen) }); + assert.equal(seen.command, join(root, "node_modules/.bin/e2e"), "npx would go to the network for a missing package"); + assert.deepEqual(seen.args, ["run", FILE, "--reporter", "json"]); + assert.equal(seen.cwd, root); + assert.equal(result.status, "passed"); + assert.deepEqual(result.steps.map(step => step.line), [8]); + assert.equal(result.output, undefined, "there are lines to mark, so the raw output is not needed"); +}); + +test("play until here runs a truncated copy beside the test, and takes it away again", async () => { + const root = await project(); + await mkdir(join(root, "tests"), { recursive: true }); + await writeFile(join(root, FILE), "the whole flow\n"); + const until = untilTestPath(FILE); + assert.equal(until, "tests/a-recorded-flow.mobile-dev-until.e2e.ts", "a sibling, and still a test the project's own patterns discover"); + // The report names the file the runner ran, which is the copy; its lines are the real file's. + const document = JSON.stringify(report([{ line: 8, status: "passed" }])).replaceAll(FILE, until); + const seen: { command?: string; args?: readonly string[]; cwd?: string } = {}; + let copy: string | undefined; + const result = await runRecordedTest({ projectPath: root, relativePath: FILE, untilSource: "the flow, cut short\n" }, { + spawner: ((command: string, args: readonly string[], options: { cwd?: string }) => { + // Read while the run is in flight: afterwards it is gone, which is the other half of this. + copy = readFileSync(join(root, until), "utf8"); + return spawner(document, 0, seen)(command, args, options as never); + }) as never, + }); + assert.deepEqual(seen.args, ["run", until, "--reporter", "json"]); + assert.equal(copy, "the flow, cut short\n"); + assert.deepEqual(result.steps.map(step => step.line), [8], "the copy's lines are the file's lines, so no mark has to be translated"); + assert.equal(existsSync(join(root, until)), false, "the copy belongs to one run and nothing else"); + assert.equal(readFileSync(join(root, FILE), "utf8"), "the whole flow\n", "a partial run must never cut the saved test down to the part that was tried"); +}); + +test("a run that produced no report explains itself with the runner's own words", async () => { + const root = await project(); + const result = await runRecordedTest({ projectPath: root, relativePath: FILE }, { spawner: spawner("No tests matched\n", 1) }); + assert.equal(result.status, "failed"); + assert.deepEqual(result.steps, []); + assert.match(result.output ?? "", /No tests matched/); +}); + +test("a project with no runner installed says so instead of failing obscurely", async () => { + const root = await mkdtemp(join(tmpdir(), "testing-run-")); + assert.throws(() => runnerBin(root), /no e2e CLI installed/); +}); + +test("the device is released before the runner is spawned, never after", async () => { + const root = await project(); + // The one correctness point of the whole feature: agent-device allows a single session per + // device, so a recorder session still open when the runner starts fails the run with "Device is + // already in use" — a refusal the user cannot connect to anything they did. + const order: string[] = []; + await runRecordedTest({ projectPath: root, relativePath: FILE }, { + release: async () => { order.push("release"); }, + spawner: ((command: string, args: readonly string[], options: { cwd?: string }) => { + order.push("spawn"); + return spawner(JSON.stringify(report([{ line: 8, status: "passed" }])))(command, args, options as never); + }) as never, + }); + assert.deepEqual(order, ["release", "spawn"]); +}); + +test("a project with no runner is refused before the recording session is given up", async () => { + const root = await mkdtemp(join(tmpdir(), "testing-run-")); + let released = false; + await assert.rejects(runRecordedTest({ projectPath: root, relativePath: FILE }, { + release: async () => { released = true; }, + spawner: spawner(""), + }), /no e2e CLI installed/); + assert.equal(released, false, "a run that cannot start must not cost the user their recording"); +}); + +test("the runner is not handed the app's pinned agent-device state directory", () => { + // The server pins one state directory so its own daemons are peers. The project's runner is a + // different build of agent-device, and pointing it at the app's directory makes it refuse before + // it reaches a test: "Daemon (pid …, v0.21.33) is newer than this client (v0.21.20)". Run exists + // to be the run the user would get from a terminal, so it gets a terminal's state directory. + const environment = runnerEnv({ PATH: "/usr/bin", AGENT_DEVICE_STATE_DIR: "/app/state", HOME: "/home/someone" }); + assert.equal("AGENT_DEVICE_STATE_DIR" in environment, false); + assert.equal(environment.PATH, "/usr/bin", "everything else the project needs is still there"); + assert.equal(environment.HOME, "/home/someone"); + assert.equal(environment.NO_COLOR, "1", "the json reporter's output must not carry escape codes"); +}); + +test("the generated config adds a reporter and changes nothing else", () => { + const source = progressConfigSource("/app/e2e.config.ts"); + // The project's own config is spread, so its targets, agents and workers all stand. + assert.match(source, /import base from "file:\/\/\/app\/e2e\.config\.ts";/); + assert.match(source, /\.\.\.resolved,/); + // A config can export a function; the runner resolves one, so the generated config does too. + assert.match(source, /typeof base === "function" \? await base\(\) : base/); + // Its own reporters come first: `--reporter` replaces the built-ins, never the configured objects. + assert.match(source, /\.\.\.\(resolved\.reporters \?\? \[\]\),/); + assert.match(source, /name: "mobile-dev-progress"/); + // Construction must have no side effects — workers load the config too — so the only write is + // inside onEvent, wrapped, because a reporter that throws is quarantined for the rest of the run. + const constructed = source.slice(0, source.indexOf("onEvent")); + assert.doesNotMatch(constructed, /process\.stderr/); + assert.match(source, /try \{\s*\n\s*process\.stderr\.write/); +}); + +test("live step events are read back off a stream the runner also writes to", () => { + const line = '@@mobile-dev-step {"stepIndex":2,"phase":"end","api":"locator.tap","label":"getByText(\\"Open\\")","status":"failed","error":"Timed out\\nat tests"}'; + assert.deepEqual(parseProgressLine(line), { + stepIndex: 2, phase: "end", api: "locator.tap", label: 'getByText("Open")', + status: "failed", error: "Timed out", + }); + // Everything else on stderr is the runner's own and must pass through untouched. + assert.equal(parseProgressLine("ℹ booting 3ECD29F0 (1 of 1)"), undefined); + assert.equal(parseProgressLine("@@mobile-dev-step not json"), undefined); + // A phase with no identity cannot be placed on a line, so it is not a mark. + assert.equal(parseProgressLine('@@mobile-dev-step {"phase":"start","api":"locator.tap"}'), undefined); + assert.equal(parseProgressLine('@@mobile-dev-step {"stepIndex":0,"phase":"activity","api":"agent.act"}'), undefined); +}); + +test("a run with a config path asks the runner to use the generated one", async () => { + const root = await project(); + const seen: { args?: readonly string[] } = {}; + await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") }, { + spawner: spawner(JSON.stringify(report([{ line: 8, status: "passed" }])), 0, seen), + onProgress: () => undefined, + }); + // Beside the project's own config, because the runner takes the project root from the config's + // own directory — one level down and the run ends in `NO_TESTS ... under <project>/.e2e`. + assert.deepEqual(seen.args, ["run", FILE, "--reporter", "json", "--config", "e2e.config.mobile-dev-progress.mts"]); + // The generated config is the run's, not the project's: it is gone once the run is. + assert.equal(existsSync(join(root, "e2e.config.mobile-dev-progress.mts")), false); +}); + +test("without a progress sink the run is left exactly as it was", async () => { + const root = await project(); + const seen: { args?: readonly string[] } = {}; + await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") }, + { spawner: spawner(JSON.stringify(report([{ line: 8, status: "passed" }])), 0, seen) }); + assert.deepEqual(seen.args, ["run", FILE, "--reporter", "json"], "no listener, so nothing is generated"); +}); + +test("a config the runner will not take costs the progress, not the run", async () => { + const root = await project(); + const calls: (readonly string[])[] = []; + const spawnBoth = ((command: string, args: readonly string[], options: { cwd?: string }) => { + calls.push(args); + // First attempt: the runner refuses the generated config. Second: the plain run, which works. + const body = args.includes("--config") + ? "INVALID_CONFIG: could not load config e2e.config.mobile-dev-progress.mts\n" + : JSON.stringify(report([{ line: 8, status: "passed" }])); + return spawner(body, args.includes("--config") ? 2 : 0)(command, args, options as never); + }) as never; + + const result = await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") }, + { spawner: spawnBoth, onProgress: () => undefined }); + + assert.equal(calls.length, 2, "the run is retried the way it worked before live progress existed"); + assert.equal(calls[1].includes("--config"), false); + assert.equal(result.status, "passed"); + assert.deepEqual(result.steps.map(step => step.line), [8]); +}); + +test("a test that really failed is not retried into a different answer", async () => { + const root = await project(); + const calls: (readonly string[])[] = []; + const failing = ((command: string, args: readonly string[], options: { cwd?: string }) => { + calls.push(args); + return spawner(JSON.stringify(report([{ line: 8, status: "failed" }], "failed")), 1)(command, args, options as never); + }) as never; + + const result = await runRecordedTest({ projectPath: root, relativePath: FILE, configPath: join(root, "e2e.config.ts") }, + { spawner: failing, onProgress: () => undefined }); + + assert.equal(calls.length, 1, "there are steps, so the config was fine and the failure is the test's"); + assert.equal(result.status, "failed"); +}); + +test("the generated config is syntactically whole, line by line", () => { + // It is source written by source, so an escape that survives one level and not the other produces + // a file the runner refuses with `CONFIG_LOAD_FAILED: Unterminated string literal` — and a run + // that cannot start. Every line must close every string it opens. + const source = progressConfigSource("/app/e2e.config.ts"); + for (const [index, line] of source.split("\n").entries()) { + const quotes = (line.match(/(?<!\\)"/g) ?? []).length; + assert.equal(quotes % 2, 0, `line ${index + 1} leaves a string open: ${line}`); + } + // The newline the reporter writes has to reach the file as an escape, not as a newline. + assert.match(source, /\}\) \+ "\\n"\);/); +}); + +test("the run's own duration comes from the report, with the attempts as the fallback", () => { + const run = (fields: Record<string, unknown>) => ({ schemaVersion: "report-1", run: { status: "passed", results: [], ...fields } }); + // `run.startedAt` is required and `run.finishedAt` is optional, so the span between them is the + // number whenever the run reached its end. + assert.equal(reportDuration(run({ startedAt: "2026-01-01T00:00:00.000Z", finishedAt: "2026-01-01T00:00:08.410Z" })), 8410); + // Cut short, so there is no `finishedAt` — but `attempt.durationMs` is required of every attempt, + // and their sum is the time the runner actually spent. + assert.equal(reportDuration(run({ + startedAt: "2026-01-01T00:00:00.000Z", + results: [{ attempts: [{ index: 0, durationMs: 1200 }, { index: 1, durationMs: 300 }] }, { attempts: [{ index: 0, durationMs: 40 }] }], + })), 1540); + // A clock that went backwards is not a duration; the attempts answer instead. + assert.equal(reportDuration(run({ + startedAt: "2026-01-01T00:00:09.000Z", finishedAt: "2026-01-01T00:00:00.000Z", + results: [{ attempts: [{ index: 0, durationMs: 7 }] }], + })), 7); + // Nothing to go on at all is left undefined rather than reported as a run that took no time. + assert.equal(reportDuration(run({ startedAt: "not a date" })), undefined); + assert.equal(reportDuration(run({ startedAt: "2026-01-01T00:00:00.000Z", results: [{ attempts: [{ index: 0 }] }] })), undefined); + assert.equal(reportDuration(undefined), undefined); + // The whole path, as the tool returns it: the steps' own durations are parsed too, and are + // there if the gutter ever wants them. + assert.equal(reportSteps(report([{ line: 8, status: "passed" }]), FILE)[0].durationMs, 12); +}); + +test("the selected target reaches the runner, so an Android recording does not run on iOS", async () => { + const root = await project(); + const seen: { command?: string; args?: readonly string[]; cwd?: string } = {}; + await runRecordedTest({ projectPath: root, relativePath: FILE, target: "android" }, + { spawner: spawner(JSON.stringify(report([{ line: 8, status: "passed" }])), 0, seen) }); + assert.deepEqual(seen.args, ["run", FILE, "--reporter", "json", "--target", "android"]); +}); diff --git a/packages/mcp-app/src/components/workspace.tsx b/packages/mcp-app/src/components/workspace.tsx index ab061025..7f6cbaaf 100644 --- a/packages/mcp-app/src/components/workspace.tsx +++ b/packages/mcp-app/src/components/workspace.tsx @@ -1,4 +1,4 @@ -import { ActivityIcon, DatabaseIcon, GitForkIcon, TerminalIcon, PanelsTopLeftIcon, ChartNoAxesCombinedIcon, NetworkIcon } from "lucide-react"; +import { ActivityIcon, DatabaseIcon, GitForkIcon, TerminalIcon, PanelsTopLeftIcon, ChartNoAxesCombinedIcon, FlaskConicalIcon, NetworkIcon } from "lucide-react"; import { useEffect, useRef, useState, useSyncExternalStore } from "react"; import type { ReactNode } from "react"; import type { ToolName, WorkspaceTools } from "@mobile-dev/ui/workspace-tools.ts"; @@ -18,6 +18,7 @@ import { setUiSurface, setUiTelemetryContext } from "@mobile-dev/ui/telemetry.ts import type { RecordingController } from "@mobile-dev/ui/recording-controller.ts"; import { RecordingCard } from "@mobile-dev/ui/components/recording-card.tsx"; import { AppFlowView } from "@mobile-dev/ui/components/app-flow-view.tsx"; +import { TestingView } from "@mobile-dev/ui/components/testing-view.tsx"; import type { DeviceLayout } from "@mobile-dev/ui/device-layout.ts"; export type { DeviceLayout } from "@mobile-dev/ui/device-layout.ts"; @@ -30,6 +31,7 @@ const tools = { "app-intel": { id: "tool-app-intel", label: "App Intel", icon: NetworkIcon }, tracesift: { id: "tool-tracesift", label: "TraceSift", icon: ChartNoAxesCombinedIcon }, "app-flow": { id: "tool-app-flow", label: "App Flow", icon: GitForkIcon }, + testing: { id: "tool-testing", label: "Testing", icon: FlaskConicalIcon }, }; type Tool = keyof typeof tools; const isTool = (value: string): value is Tool => value in tools; @@ -38,6 +40,7 @@ export function Workspace({ panels, recordingController, banner, displayModeCont const { logs, performance, storage, tracesift } = panels; const appIntel = panels["app-intel"]; const appFlow = panels["app-flow"]; + const testing = panels.testing; const [tool, setTool] = useState<Tool>("logs"); const [savedVisible, setSavedVisible] = useState(false); const [simulatorsCollapsed, setSimulatorsCollapsed] = useState(false); @@ -58,6 +61,15 @@ export function Workspace({ panels, recordingController, banner, displayModeCont onLayout("none"); panels.hide(); }, [recordingState.recording?.id]); + // An agent recording opens the Testing tab for the user: they asked their agent for a test, and + // the steps appearing one at a time are the whole point. Which tool is on screen is this + // component's own state, so the panel asks rather than reaches. + useEffect(() => testing.onAgentRecordingStarted(() => { + manualLayout.current = true; + setTool("testing"); + setSavedVisible(false); + panels.show("testing"); + }), [testing]); const performanceState = useSyncExternalStore(performance.subscribe, performance.getSnapshot); const wide = useMediaQuery("(min-width: 900px)"); const fullscreen = document.documentElement.dataset.view === "workspace"; @@ -68,12 +80,26 @@ export function Workspace({ panels, recordingController, banner, displayModeCont const traceSiftState = useSyncExternalStore(tracesift.subscribe, tracesift.getSnapshot); const appIntelState = useSyncExternalStore(appIntel.subscribe, appIntel.getSnapshot); const appFlowState = useSyncExternalStore(appFlow.subscribe, appFlow.getSnapshot); + const testingState = useSyncExternalStore(testing.subscribe, testing.getSnapshot); // A saved recording shows in the performance tool, so it counts as that tool being open. - const open = { logs: logState.open, performance: savedVisible || performanceState.open, storage: storageState.open, "app-intel": appIntelState.open, tracesift: traceSiftState.open, "app-flow": appFlowState.open }[tool]; + const open = { logs: logState.open, performance: savedVisible || performanceState.open, storage: storageState.open, "app-intel": appIntelState.open, tracesift: traceSiftState.open, "app-flow": appFlowState.open, testing: testingState.open }[tool]; useEffect(() => { setUiSurface(open ? savedVisible ? "recording" : tool : "simulator"); }, [tool, open, savedVisible]); useEffect(() => { setUiTelemetryContext({ logs_open: logState.open, logs_paused: logState.paused, performance_running: performanceState.monitoring || performanceState.fpsMonitoring }); }, [logState.open, logState.paused, performanceState.monitoring, performanceState.fpsMonitoring]); + const testingOpen = tool === "testing" && open === true; + /** + * The Testing panel is read as three panes side by side — mirror, preview, steps — so it asks + * for a horizontal workspace of its own rather than inheriting `split`. + * + * `split` is `wide && fullscreen`, and outside the fullscreen workspace it is false: the group + * was then vertical, the mirror kept the 60% default meant for a tool stacked under it, and the + * Testing panel's own two panes were left with 40% of the height between them. Recording is done + * by tapping the mirror and read back beside it, which only works as a row — so wherever there + * is width for a row, that is what this panel gets. + */ + const testingSplit = testingOpen && wide; + const sideBySide = split || testingSplit; const isOpen = () => (tool === "performance" && savedVisible) || panels.panel(tool).getSnapshot().open; const closeTools = () => { setSavedVisible(false); @@ -87,6 +113,7 @@ export function Workspace({ panels, recordingController, banner, displayModeCont }; useEffect(() => { if (automaticTool) { setTool(automaticTool); setSavedVisible(false); panels.show(automaticTool); } }, [automaticTool, automaticToolRevision]); const logsRef = usePanelRef(); + const simulatorsRef = usePanelRef(); const simulatorGroupRef = useGroupRef(); const workspaceGroupRef = useGroupRef(); useEffect(() => { if (open) logsRef.current?.expand(); else logsRef.current?.collapse(); }, [open, split, layout]); @@ -110,7 +137,21 @@ export function Workspace({ panels, recordingController, banner, displayModeCont const panelIds = Object.keys(current); if (panelIds.length > 0) { workspacePending = false; - workspace.setLayout({ "tools-resizable": layout === "none" ? 100 : split ? 64 : 40, "simulators-resizable": layout === "none" ? 0 : split ? 36 : 60 }); + // The Testing panel is read as three panes — mirror, preview, steps — and the preview + // and the steps split the tools pane 60/40 of their own accord. A quarter of the + // workspace for the mirror is what makes those 45% and 30%: the mirror is tapped, while + // the two beside it are read, and reading a generated file needs the room. + const toolsShare = testingOpen ? 75 : split ? 64 : 40; + workspace.setLayout({ "tools-resizable": toolsShare, "simulators-resizable": 100 - toolsShare }); + // No visible platform is the panel being collapsed, not a share of 0. A share is the + // group's last *requested* layout, which it re-applies whenever a panel's constraints + // change — and the Testing panel changes them as it opens. A 0 requested while nothing + // was bootable therefore outlived the device: discovery selected a simulator, the + // platform selector read "iOS", and the mirror stayed shut until the user toggled the + // platform off and on again. Collapsing says the same thing in the group's own terms, + // so expanding it is all that finding a device has to do. + if (layout === "none") simulatorsRef.current?.collapse(); + else simulatorsRef.current?.expand(); if (isOpen() === false) logsRef.current?.collapse(); } } @@ -120,18 +161,28 @@ export function Workspace({ panels, recordingController, banner, displayModeCont observer.observe(document.getElementById("simulator-panels")!); observer.observe(document.getElementById("workspace-panels")!); applyLayout(); - return () => observer.disconnect(); - }, [layout, split, tool]); - const toolPanel = <ResizablePanel key="tools" id="tools-resizable" panelRef={logsRef} defaultSize={split ? "64%" : "40%"} minSize={split ? "240px" : "25%"} maxSize="100%" collapsible collapsedSize={`${collapsedToolSize}px`} onResize={(size, _id, previous) => { + // And again on the next frame, because the group re-derives its own layout from the panels' + // defaults when their constraints change — which is what opening the Testing panel does, since + // its mirror and its tools declare sizes of their own. Asking during the commit is asking + // before that, so the answer was thrown away; the frame after it is when it sticks. + const frame = requestAnimationFrame(() => { workspacePending = true; simulatorsPending = layout !== "none"; applyLayout(); }); + return () => { cancelAnimationFrame(frame); observer.disconnect(); }; + }, [layout, split, sideBySide, tool, testingOpen]); + const toolPanel = <ResizablePanel key="tools" id="tools-resizable" panelRef={logsRef} defaultSize={testingOpen ? "75%" : split ? "64%" : "40%"} minSize={sideBySide ? "240px" : "25%"} maxSize="100%" collapsible collapsedSize={`${collapsedToolSize}px`} onResize={(size, _id, previous) => { if (!previous) return; if (size.inPixels <= collapsedToolSize + 1 && isOpen()) closeTools(); else if (size.inPixels > 80 && isOpen() === false) showTool(tool); - }}>{tool === "logs" && <LogsView panel={logs} />}{tool === "storage" && <StorageView panel={storage} />}{tool === "app-intel" && <AppIntelligenceView panel={appIntel} />}{tool === "tracesift" && <TraceSiftView panel={tracesift} />}{tool === "app-flow" && open && <AppFlowView panel={appFlow} />}{tool === "performance" && open && (savedVisible ? <div className="recording-workspace"><Button variant="outline" size="sm" onClick={() => { setSavedVisible(false); performance.show(); }}>Live performance</Button><RecordingCard controller={recordingController} detailed /></div> : <PerformanceView panel={performance} />)}</ResizablePanel>; + }}>{tool === "logs" && <LogsView panel={logs} />}{tool === "storage" && <StorageView panel={storage} />}{tool === "app-intel" && <AppIntelligenceView panel={appIntel} />}{tool === "tracesift" && <TraceSiftView panel={tracesift} />}{tool === "app-flow" && open && <AppFlowView panel={appFlow} />}{tool === "testing" && open && <TestingView panel={testing} deviceApps={panels.deviceApps} />}{tool === "performance" && open && (savedVisible ? <div className="recording-workspace"><Button variant="outline" size="sm" onClick={() => { setSavedVisible(false); performance.show(); }}>Live performance</Button><RecordingCard controller={recordingController} detailed /></div> : <PerformanceView panel={performance} />)}</ResizablePanel>; // The sidebar keeps these controls under its simulators. Fullscreen, or no visible // platform, moves them above the tools. const bottomToolbar = !fullscreen && layout !== "none"; const menuSide = bottomToolbar ? "top" : "bottom"; const ToolIcon = tools[tool].icon; + // The Testing panel records by tapping the mirror and reads the test it emits beside it, so the + // mirror leads and the preview follows it — the reading order of the thing being done. Every + // other tool is a readout the simulator is secondary to, and keeps the simulator on the right. + // Only side by side: stacked, "left" is the top panel, which the simulator already is. + const mirrorFirst = testingOpen && sideBySide; const toolbar = <nav className={`workspace-toolbar @container flex h-12 shrink-0 items-center justify-between gap-2 px-2 ${bottomToolbar ? "border-t" : "border-y"}`} aria-label="Developer tools"> {displayModeControl} <Select items={[{ value: "none", label: "Tools" }, ...Object.entries(tools).map(([value, { label }]) => ({ value, label }))]} value={open ? tool : "none"} onValueChange={value => { @@ -156,7 +207,7 @@ export function Workspace({ panels, recordingController, banner, displayModeCont </SelectGroup></SelectContent> </Select> </nav>; - const simulatorPanel = <ResizablePanel key="simulators" id="simulators-resizable" minSize="0%" maxSize="100%" collapsible collapsedSize="0px" defaultSize={split ? "36%" : "60%"} onResize={size => setSimulatorsCollapsed(size.inPixels < 1)}> + const simulatorPanel = <ResizablePanel key="simulators" id="simulators-resizable" panelRef={simulatorsRef} minSize="0%" maxSize="100%" collapsible collapsedSize="0px" defaultSize={testingOpen ? "25%" : split ? "36%" : "60%"} onResize={size => setSimulatorsCollapsed(size.inPixels < 1)}> <div className="flex h-full min-h-0 flex-col"> <ResizablePanelGroup id="simulator-panels" groupRef={simulatorGroupRef} orientation="horizontal" className="min-h-0 flex-1" aria-label="Simulators"> <ResizablePanel id="ios-resizable" defaultSize="100%" minSize="0px" maxSize="100%" collapsible collapsedSize="0px"><SimulatorView platform="ios" visible={layout === "ios" || layout === "both"} /></ResizablePanel> @@ -173,8 +224,8 @@ export function Workspace({ panels, recordingController, banner, displayModeCont {banner} {contextControl} {!bottomToolbar && toolbar} - <ResizablePanelGroup id="workspace-panels" groupRef={workspaceGroupRef} orientation={split ? "horizontal" : "vertical"} className="min-h-0 flex-1" data-split={split}> - {split ? [toolPanel, divider, simulatorPanel] : [simulatorPanel, divider, toolPanel]} + <ResizablePanelGroup id="workspace-panels" groupRef={workspaceGroupRef} orientation={sideBySide ? "horizontal" : "vertical"} className="min-h-0 flex-1" data-split={sideBySide}> + {sideBySide ? (mirrorFirst ? [simulatorPanel, divider, toolPanel] : [toolPanel, divider, simulatorPanel]) : [simulatorPanel, divider, toolPanel]} </ResizablePanelGroup> </main>; } diff --git a/packages/shared/src/panel-target.ts b/packages/shared/src/panel-target.ts index fb08cff5..13a65a0c 100644 --- a/packages/shared/src/panel-target.ts +++ b/packages/shared/src/panel-target.ts @@ -1,17 +1,33 @@ import { z } from "zod"; +import type { SimulatorDevice } from "./protocol.ts"; const platform = z.enum(["ios", "android"]).optional().describe("The task's established platform. Omit when undecided."); const kind = z.enum(["simulator", "emulator", "physical"]).optional().describe("The requested device kind. Omit when undecided."); -const deviceId = z.string().min(1).max(256).regex(/^[A-Za-z0-9_.:\[\]%-]+$/).optional().describe("An exact device ID from device discovery. Omit to select a matching device automatically."); +export const deviceIdSchema = z.string().min(1).max(256).regex(/^[A-Za-z0-9_.:\[\]%-]+$/).describe("An exact device ID from device discovery. Omit to select a matching device automatically."); const workspaceTool = z.enum(["logs", "performance", "storage", "app-intel", "tracesift"]).optional().describe("A workspace tool to open. Omit for the device view."); const appIntelScreen = z.string().min(1).max(512).optional().describe("An App Intelligence screen ID to focus when workspaceTool is app-intel, from mobile_app_intel's search or screen result. Omit to open on whatever screen the panel last showed."); -export const panelTargetSchema = z.object({ platform, kind, deviceId, workspaceTool, appIntelScreen }).strict().superRefine((target, context) => { +export const panelTargetSchema = z.object({ platform, kind, deviceId: deviceIdSchema.optional(), workspaceTool, appIntelScreen }).strict().superRefine((target, context) => { if (target.platform === "ios" && target.kind === "emulator" || target.platform === "android" && target.kind === "simulator") { context.addIssue({ code: "custom", path: ["kind"], message: "Use simulator for iOS or emulator for Android." }); } }); +/** + * Whether the Testing panel can record against a device: a booted iOS simulator or a booted + * Android emulator. Anything physical is out, because agent-device's session on a physical + * device is not the one the mirror holds. + */ +/** The platform the recorder attaches with. A simulator listing that omits `platform` is iOS. */ +export function recordablePlatform(device: SimulatorDevice): "ios" | "android" { + return device.platform === "android" ? "android" : "ios"; +} + +export function isRecordableDevice(device?: SimulatorDevice): boolean { + if (!device || device.state !== "Booted" || device.kind === "physical") return false; + return device.platform === "android" ? device.kind === "emulator" : device.platform === "ios" || device.platform === undefined; +} + export type PanelTarget = z.infer<typeof panelTargetSchema>; export function panelTargetPlatform(target: PanelTarget): "ios" | "android" | undefined { diff --git a/packages/shared/src/screen-annotations.ts b/packages/shared/src/screen-annotations.ts index 2e18dc01..18f44aaf 100644 --- a/packages/shared/src/screen-annotations.ts +++ b/packages/shared/src/screen-annotations.ts @@ -4,7 +4,16 @@ export type ScreenPoint = { x: number; y: number }; export type ScreenSize = { width: number; height: number }; export type ScreenBounds = ScreenPoint & ScreenSize; export type ReactElementContext = { component: string; owners: string[]; key?: string; sourceKind?: "element" | "owner"; source?: { file: string; line: number; column?: number; functionName?: string } }; -export type ScreenComponent = { name: string; bounds: ScreenBounds; role?: string; identifier?: string; label?: string; value?: string; depth: number; source?: "accessibility" | "screen" | "react-native"; nodeId?: string; parentId?: string; react?: ReactElementContext }; +/** + * One element over the screen's pixels. + * + * `role`, `label`, `text`, `value`, `placeholder` and `hidden` are the test runner's own fields, + * carried through untouched when the recorder reads the runner's tree: `label` is the accessible + * *name* `getByRole`/`getByLabel` match, `text` is what `getByText` also searches, and `hidden` + * is the node a role query skips, and `disabled`/`selected` are the states `toBeDisabled` and + * `toBeSelected` read. An inspection that does not report one simply leaves it unset. + */ +export type ScreenComponent = { name: string; bounds: ScreenBounds; role?: string; identifier?: string; label?: string; text?: string; value?: string; placeholder?: string; hidden?: boolean; disabled?: boolean; selected?: boolean; depth: number; source?: "accessibility" | "screen" | "react-native"; nodeId?: string; parentId?: string; react?: ReactElementContext }; export type ScreenSelectionContext = { ancestors: ScreenComponent[]; siblings: ScreenComponent[]; siblingCount: number; children: ScreenComponent[]; childCount: number; instance?: { index: number; total: number } }; export type ScreenAnnotation = { id: string; @@ -59,6 +68,31 @@ export function accessibilityScreen(tree: unknown): ScreenSize | undefined { if (frame?.x === 0 && frame.y === 0 && /^(AX)?Application$/i.test(String(root?.role ?? ""))) return screenSize(frame); } +/** + * The same components, in another screen's coordinate space. + * + * The recorder's tree and the mirror do not always measure in the same unit. An iOS simulator + * reports both in points, so nothing happens here. Android reports element bounds in device + * pixels while scrcpy caps the stream's longest edge — a 1280x2856 emulator arrives as 574x1280 — + * so every rectangle is more than twice the size of the picture the user is pointing at, and a + * hit test finds only the full-screen groups. One boundary converts; the hit ladder, the outline + * and the tap that follows all stay in the mirror's own space. + */ +export function scaledComponents(tree: readonly ScreenComponent[], from: ScreenSize, to: ScreenSize): ScreenComponent[] { + if (!(from.width > 0) || !(from.height > 0) || !(to.width > 0) || !(to.height > 0)) return [...tree]; + const scaleX = to.width / from.width, scaleY = to.height / from.height; + // An exact match is every iOS read and a native-resolution Android stream; copying 10,000 + // rectangles to multiply them by one is work with nothing to show for it. + if (Math.abs(scaleX - 1) < 1e-6 && Math.abs(scaleY - 1) < 1e-6) return [...tree]; + return tree.map(component => ({ + ...component, + bounds: { + x: component.bounds.x * scaleX, y: component.bounds.y * scaleY, + width: component.bounds.width * scaleX, height: component.bounds.height * scaleY, + }, + })); +} + // Accept backend trees and normalized, flat MCP snapshots with explicit parents. export function screenComponents(tree: unknown, scale = 1): ScreenComponent[] { const components: ScreenComponent[] = []; diff --git a/packages/shared/src/telemetry.ts b/packages/shared/src/telemetry.ts index 4eaeed1f..e1d93ebd 100644 --- a/packages/shared/src/telemetry.ts +++ b/packages/shared/src/telemetry.ts @@ -41,7 +41,7 @@ export function validateTelemetryEnvironment(value: unknown): TelemetryEnvironme throw new Error("Sentry environment must be development or release."); } -export type Surface = "app-flow" | "logs" | "performance" | "storage" | "simulator" | "recording" | "comparison" | "tracesift" | "app-intel"; +export type Surface = "app-flow" | "logs" | "performance" | "storage" | "simulator" | "recording" | "comparison" | "tracesift" | "app-intel" | "testing"; export type TelemetryAttributes = Record<string, string | number | boolean>; type Options = NonNullable<Parameters<typeof init>[0]>; type ErrorEvent = Parameters<NonNullable<Options["beforeSend"]>>[0]; diff --git a/packages/shared/src/test-drafts.ts b/packages/shared/src/test-drafts.ts new file mode 100644 index 00000000..3bb42693 --- /dev/null +++ b/packages/shared/src/test-drafts.ts @@ -0,0 +1,114 @@ +import { z } from "zod"; +import type { RecordedStep } from "./test-recording.ts"; + +/** + * A recording that has not reached the project yet. + * + * The recorder holds one flow at a time and holds it in the panel document, which the host + * rebuilds whenever the user looks at something else. Until now that meant a flow the user had + * not saved — or one their agent recorded while they were reading the chat — was simply gone when + * they came back. A draft is that flow written down: the steps as data, not the emitted source, + * so it comes back editable rather than as a file the panel cannot take apart again. + * + * A preview code edit adds `source`; that source then takes precedence over generated steps. + * Drafts are the plugin's own state, not the project's: nothing is ever written into the user's + * repository until they press Save. One is deleted when its flow is saved, and when the user says + * so. See `packages/core/src/testing-drafts.ts`. + */ + +const point = z.object({ x: z.number().finite(), y: z.number().finite() }).strict(); + +const namedSelectorSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("testId"), testId: z.string().max(1024) }).strict(), + z.object({ kind: z.literal("role"), role: z.string().max(256), name: z.string().max(1024) }).strict(), + z.object({ kind: z.literal("placeholder"), placeholder: z.string().max(1024) }).strict(), + z.object({ kind: z.literal("text"), text: z.string().max(1024) }).strict(), + z.object({ kind: z.literal("label"), label: z.string().max(1024) }).strict(), +]); + +export const recordedSelectorSchema = z.union([ + namedSelectorSchema, + z.object({ kind: z.literal("point"), point }).strict(), + z.object({ + kind: z.literal("nth"), + inner: namedSelectorSchema, + index: z.number().int().min(0).max(4096), + matches: z.number().int().min(0).max(4096), + }).strict(), +]); + +const stepTargetSchema = z.object({ + role: z.string().max(256).optional(), + placeholder: z.string().max(1024).optional(), + label: z.string().max(1024).optional(), + text: z.string().max(1024).optional(), + value: z.string().max(4096).optional(), +}).strict(); + +const pinnedStepKindSchema = z.enum([ + "tap", "doubleTap", "longPress", "fill", "scrollUntilVisible", + "assertVisible", "assertNotVisible", "assertText", "assertContainsText", "assertValue", + "assertEnabled", "assertDisabled", "assertSelected", "swipe", +]); + +const visionSchema = z.union([z.boolean(), z.literal("only")]); +const instruction = z.string().max(4096); +const stepId = z.string().min(1).max(128); +const stepLabel = z.string().max(1024); + +export const recordedStepSchema = z.union([ + z.object({ + id: stepId, + label: stepLabel, + kind: pinnedStepKindSchema, + selector: recordedSelectorSchema, + value: z.string().max(4096).optional(), + direction: z.enum(["up", "down", "left", "right"]).optional(), + ambiguous: z.boolean().optional(), + anchor: recordedSelectorSchema.optional(), + target: stepTargetSchema.optional(), + }).strict(), + z.object({ + id: stepId, label: stepLabel, kind: z.literal("agentAct"), instruction, + params: z.record(z.string().max(256), z.string().max(4096)).optional(), + }).strict(), + z.object({ id: stepId, label: stepLabel, kind: z.literal("agentAssert"), instruction, vision: visionSchema.optional() }).strict(), + z.object({ id: stepId, label: stepLabel, kind: z.literal("agentWaitFor"), instruction, vision: visionSchema.optional() }).strict(), +]); + +/** The most steps one draft may hold, well past the "a long flow is two tests" guidance. */ +export const MAX_DRAFT_STEPS = 200; + +export const testDraftIdSchema = z.uuid(); + +export const testDraftSchema = z.object({ + schemaVersion: z.literal(1), + id: testDraftIdSchema, + /** The project the flow was recorded against, so the panel only lists its own drafts. */ + projectPath: z.string().min(1).max(4096), + title: z.string().max(160), + /** The file name the user typed under `tests/`, or "" to derive it from the title on save. */ + fileName: z.string().max(256), + /** Who recorded it, for the row's label. */ + origin: z.enum(["user", "agent"]), + steps: z.array(recordedStepSchema).max(MAX_DRAFT_STEPS), + /** Source edited in the preview. When present it takes precedence over generated steps. */ + source: z.string().max(256 * 1024).optional(), + createdAt: z.iso.datetime(), + updatedAt: z.iso.datetime(), +}).strict(); + +export type TestDraft = Omit<z.infer<typeof testDraftSchema>, "steps"> & { steps: RecordedStep[] }; + +/** What a draft row needs: everything but the steps, which only the opened draft reads. */ +export type TestDraftSummary = Omit<TestDraft, "steps"> & { stepCount: number }; + +export function draftSummary(draft: TestDraft): TestDraftSummary { + const { steps, source, ...rest } = draft; + return { ...rest, stepCount: source ? source.split("\n").filter(line => /^\s*await\s+/.test(line) && !/^\s*await\s+app\.open\s*\(/.test(line)).length : steps.length }; +} + +/** What a draft reads as in the list when it has no title of its own. */ +export function draftLabel(draft: Pick<TestDraft, "title">): string { + return draft.title.trim() || "Untitled flow"; +} diff --git a/packages/shared/src/test-recording.ts b/packages/shared/src/test-recording.ts new file mode 100644 index 00000000..597f9b37 --- /dev/null +++ b/packages/shared/src/test-recording.ts @@ -0,0 +1,1010 @@ +import { componentsAt } from "./screen-annotations.ts"; +import type { ScreenBounds, ScreenComponent, ScreenPoint, ScreenSize } from "./screen-annotations.ts"; + +/** + * How a recorded step names the element it acted on. The ranks are tried in order and the first + * one that resolves to exactly one element on screen wins, so a recorded test is pinned to + * something stable rather than to a coordinate. + * + * `point` is the fallback: it always works and it is always brittle, so the emitter marks it. + */ +export type RecordedSelector = + | { kind: "testId"; testId: string } + | { kind: "role"; role: string; name: string } + | { kind: "placeholder"; placeholder: string } + | { kind: "text"; text: string } + | { kind: "label"; label: string } + | { kind: "point"; point: ScreenPoint } + /** + * A named selector narrowed to one of its matches by position. + * + * The last resort before a coordinate, and far better than one: when nothing on the screen names + * the element uniquely — the same text on a heading and on a tab, with no distinguishing ancestor + * to scope under and nothing to `filter({ hasText })` on — a bare selector matching two is a step + * that fails with `LOCATOR_AMBIGUOUS`, while the same selector at `.nth(i)` is a step that runs. + * `index` is the position of the element the user actually pointed at, so it is recorded, not + * guessed. It is still positional, so the emitter says so and names the durable fix. + */ + | { kind: "nth"; inner: NamedSelector; index: number; matches: number }; + +/** Every selector that names something, as opposed to a coordinate or a position among names. */ +export type NamedSelector = Exclude<RecordedSelector, { kind: "point" } | { kind: "nth" }>; + +/** + * What a recorded step does to the element it named. + * + * Every one of these is an action or matcher the `e2e` runner actually has, which is the whole + * constraint: the element menu offers rows from this list, so a row the runner cannot express is + * a row that does not exist. `secondaryTap`, `selectOption`, `setInputFiles` and `scrollIntoView` + * are deliberately absent — the runner declares them, and a device refuses them with + * `UNSUPPORTED_CAPABILITY`. So are `check`/`uncheck` and `toBeChecked`: the runner has them, and + * whether a device reports a `checked` state at all depends on the platform and the component, so + * a row for them would emit a step that cannot be relied on to run. + * + * The menu shows the common few up front and the rest behind "All commands"; see `stepOffers` and + * `allStepOffers`. + */ +export type PinnedStepKind = + | "tap" + | "doubleTap" + | "longPress" + | "fill" + | "scrollUntilVisible" + | "assertVisible" + | "assertNotVisible" + | "assertText" + | "assertContainsText" + | "assertValue" + | "assertEnabled" + | "assertDisabled" + | "assertSelected" + | "swipe"; + +/** + * The four directions `screen.swipe` and `locator.swipe` take, as `e2e`'s own `ScrollDirection` + * declares them. `Momentum` is the other `SwipeOptions` key and the panel never writes it: its + * default is `none`, and how hard a flick was thrown is not something a recorded step can observe. + */ +export type SwipeDirection = "up" | "down" | "left" | "right"; +export const SWIPE_DIRECTIONS: readonly SwipeDirection[] = ["up", "down", "left", "right"]; + +/** A direction the runner understands, defaulting to the one the menu lists first. */ +export function swipeDirection(value?: string): SwipeDirection { + return value === "down" || value === "left" || value === "right" ? value : "up"; +} + +/** + * The gesture a recorded swipe takes on the device, drawn inside the box it is scoped to. + * + * Taken over the middle 70% of the box, so the start and the end are both well inside it: a + * gesture begun on the very edge of the screen is an edge gesture to the OS — back, the shade, + * the app switcher — and not the swipe the step describes. The finger travels the way the step is + * named, which is the direction `e2e` names too. + * + * It lives here rather than beside either driver because both take the step they record: the + * panel sends it through the mirror's input stream, and an agent recording sends it through + * agent-device. One arithmetic, so a swipe means the same gesture whoever asked for it. + */ +export function swipePath(box: ScreenBounds, direction: SwipeDirection): { from: ScreenPoint; to: ScreenPoint } { + const midX = box.x + box.width / 2; + const midY = box.y + box.height / 2; + const reachX = box.width * 0.35; + const reachY = box.height * 0.35; + switch (direction) { + case "up": return { from: { x: midX, y: midY + reachY }, to: { x: midX, y: midY - reachY } }; + case "down": return { from: { x: midX, y: midY - reachY }, to: { x: midX, y: midY + reachY } }; + case "left": return { from: { x: midX + reachX, y: midY }, to: { x: midX - reachX, y: midY } }; + case "right": return { from: { x: midX - reachX, y: midY }, to: { x: midX + reachX, y: midY } }; + } +} + +/** + * The two steps that are a sentence rather than a locator. + * + * `agent.act` and `agent.assert` take an instruction and nothing else — no locator, not even an + * optional one — so an agent step is a different shape of step, not a pinned one with its selector + * left out. That is the whole reason `RecordedStep` is a union: a pinned step without a selector + * has never been a legal step and must stay impossible to write. + */ +export type AgentStepKind = "agentAct" | "agentAssert" | "agentWaitFor"; + +export type RecordedStepKind = PinnedStepKind | AgentStepKind; + +/** + * What `agent.assert`'s judge is shown, as `e2e`'s own `VisionMode` declares it: the tree, the tree + * with a masked screenshot, or the screenshot alone. `false` is the runner's default, which is why + * the emitter leaves the options object off entirely rather than writing it out. + */ +export type VisionMode = boolean | "only"; + +/** What every step carries, however it names what it acts on. */ +type RecordedStepCommon = { + id: string; + /** What the step reads as in the panel's list, e.g. `Tap "Explore" (tab)`. */ + label: string; +}; + +/** A step pinned to one element, which is every step the mirror's command menu records. */ +export type PinnedStep = RecordedStepCommon & { + kind: PinnedStepKind; + selector: RecordedSelector; + /** The text a `fill` types. Only a `fill` has one. */ + value?: string; + /** Assertion wait budget in milliseconds; absent uses e2e's default. */ + timeoutMs?: number; + /** + * Which way a `swipe` goes. Only a `swipe` has one, and it is its own field rather than `value` + * because it is a choice from four, not text the user supplies — nothing asks for it through a + * form, and nothing compares it with what is on screen. + */ + direction?: SwipeDirection; + /** Set when every rank matched more than one element, so the panel can flag the step. */ + ambiguous?: boolean; + /** + * For a coordinate tap only: something on the same screen the test can wait for first. A + * coordinate is always actionable, so without this the tap fires before the screen exists. + */ + anchor?: RecordedSelector; + /** + * What the element carried when the step was recorded, so the step list can offer the same + * commands the mirror's menu did without the tree it resolved from. The screen the step was + * recorded on is long gone by the time a user re-opens that list, and re-reading the device + * would answer about a different screen; the fields a command row is gated on are few and they + * belong to the step, so they are kept with it. + */ + target?: StepTarget; +}; + +/** + * A step the agent carries out, or judges, from a sentence. + * + * The element the user tapped to record it is a typing shortcut for that sentence and nothing + * more: it is never stored here and never emitted, because the runner resolves the instruction + * against the screen it finds at run time, not against the tree the panel read. + */ +export type AgentStep = RecordedStepCommon & ( + | { kind: "agentAct"; instruction: string; params?: AgentParams } + | { kind: "agentAssert"; instruction: string; vision?: VisionMode } + | { kind: "agentWaitFor"; instruction: string; vision?: VisionMode } +); + +/** + * The values an `act` instruction refers to, as the panel records them. + * + * Every value is held as the text the user typed and turned into a literal only on emit, because + * the panel's editor is a key and a text field: there is no type picker, so the text is the only + * thing there is to store, and storing a parsed value would mean deciding its type at the moment + * of a keystroke rather than at the moment the file is written. + * + * A record rather than a list of pairs, which settles the duplicate key: `params` is an object in + * the emitted file, so two rows with one key can only ever mean the later row, and keeping a list + * would preserve a difference the runner cannot express. The editor writes the rows into this in + * order, so the last row with a given key is the one that survives. + * + * `e2e`'s own `AgentParams` admits nested objects, arrays, `Secret` and `unique()`. The panel + * records none of those: each is a decision with a runtime meaning — a secret is withheld from the + * model, a unique value defeats the replay cache — and a key/value row is not where that is + * chosen. Those stay something the user writes in the file. + */ +export type AgentParams = Record<string, string>; + +export type RecordedStep = PinnedStep | AgentStep; + +/** e2e's default assertionTimeout when the project does not override it. */ +export const DEFAULT_ASSERTION_TIMEOUT_MS = 5_000; + +export function isAgentStepKind(kind: RecordedStepKind): kind is AgentStepKind { + return kind === "agentAct" || kind === "agentAssert" || kind === "agentWaitFor"; +} + +export function isAgentStep(step: RecordedStep): step is AgentStep { + return isAgentStepKind(step.kind); +} + +/** + * The part of an element a command row is gated on: only a text field can be filled or have a + * value, and only an element with text of its own can be compared against text. It is a subset of + * `ScreenComponent` so a live component can be passed straight through. + */ +export type StepTarget = Pick<ScreenComponent, "role" | "placeholder" | "label" | "text" | "value">; + +/** + * The roles `screen.getByRole` accepts, as `e2e`'s own `Role` union declares them. + * + * This is a *filter*, never a translation. The engine has already decided what each element's role + * is — it projected the snapshot the recorder reads, and it will project the one the test runs + * against — so nothing here renames a role; `tab` reaches the file because the engine said `tab`. + * But the union is closed, so `getByRole("other", …)` or `getByRole("text", …)` is a type error in + * the emitted file rather than a query that merely misses, and the engine deliberately keeps an + * unmapped element type as its own role. Those fall through to the text and label ranks, which is + * how the element gets named instead. + */ +const QUERYABLE_ROLES = new Set([ + "button", "link", "textbox", "searchbox", "combobox", "listbox", "option", "checkbox", "radio", + "radiogroup", "switch", "slider", "spinbutton", "progressbar", "meter", "image", "heading", "tab", + "tablist", "tabpanel", "menu", "menubar", "menuitem", "menuitemcheckbox", "menuitemradio", + "toolbar", "tooltip", "tree", "treeitem", "list", "listitem", "table", "grid", "row", "rowgroup", + "rowheader", "cell", "gridcell", "columnheader", "separator", "group", "article", "figure", + "form", "status", "alert", "dialog", "alertdialog", "main", "navigation", "banner", "contentinfo", + "complementary", "region", +]); + +/** + * The runner compares text exactly, after collapsing whitespace, so both sides of a count have to + * be normalised the same way. An accessibility label carries the line breaks of the rendered copy. + */ +const normalize = (value: string | undefined) => (value ?? "").replace(/\s+/g, " ").trim(); + +/** + * The element's accessible *name*: what `getByRole(role, name)` and `getByLabel` match. + * + * It is the runner's own `name` field, carried through the read path untouched. A search field + * whose only text is its placeholder has no name at all, which is why `getByRole("textbox", + * "Search transactions")` used to be emitted and used to match nothing. + */ +function accessibleName(component: ScreenComponent): string { + return normalize(component.label); +} + +/** The element's own text, which `getByText` searches besides the name. */ +function elementText(component: ScreenComponent): string { + return normalize(component.text); +} + +/** Parent links by node id, for the containment the runner's innermost-only rule needs. */ +function ancestry(components: ScreenComponent[]): Map<string, string> { + const parents = new Map<string, string>(); + for (const item of components) if (item.nodeId && item.parentId) parents.set(item.nodeId, item.parentId); + return parents; +} + +function isWithin(parents: Map<string, string>, entry: ScreenComponent, ancestor: ScreenComponent): boolean { + if (!entry.nodeId || !ancestor.nodeId) return false; + for (let id = parents.get(entry.nodeId); id !== undefined; id = parents.get(id)) { + if (id === ancestor.nodeId) return true; + } + return false; +} + +/** + * Drops every match that contains another match, which is how the runner answers `getByText` and + * `getByLabel`. + * + * A device tree echoes text upwards: iOS reports a React Native `Text` host view and the + * `StaticText` inside it with the same label, so without this rule a screen's every piece of text + * is "ambiguous" and every step records a TODO. The runner does not see it that way, and a + * recorder that disagrees with the runner is the bug. + */ +function innermostOnly(components: ScreenComponent[], matches: ScreenComponent[]): ScreenComponent[] { + if (matches.length < 2) return matches; + const parents = ancestry(components); + return matches.filter(entry => !matches.some(other => other !== entry && isWithin(parents, other, entry))); +} + +/** + * The elements a selector matches, in the order the runner enumerates them. + * + * Ordered, not counted, because `.nth(i)` needs the position of one particular element among them — + * the one the user pointed at. The order is the snapshot's own: the projection flattens the tree + * depth first, and the panel passes it through untouched, so the panel's index and the runner's + * index are the same index. + */ +function matchesTestId(components: ScreenComponent[], testId: string): ScreenComponent[] { + return components.filter(item => normalize(item.identifier) === normalize(testId)); +} + +/** A role query compares the role exactly and skips a hidden node, as the runner does. */ +function matchesRole(components: ScreenComponent[], role: string, name: string): ScreenComponent[] { + return components.filter(item => (item.role ?? "") === role && accessibleName(item) === name && item.hidden !== true); +} + +function matchesPlaceholder(components: ScreenComponent[], placeholder: string): ScreenComponent[] { + return components.filter(item => normalize(item.placeholder) === placeholder); +} + +function matchesText(components: ScreenComponent[], text: string): ScreenComponent[] { + return innermostOnly(components, components.filter(item => accessibleName(item) === text || elementText(item) === text)); +} + +function matchesLabel(components: ScreenComponent[], label: string): ScreenComponent[] { + return innermostOnly(components, components.filter(item => accessibleName(item) === label)); +} + +/** + * The single piece of text inside an element, when there is exactly one. + * + * More than one and there is nothing to choose between them, so the element stays unnamed and the + * ladder moves outwards instead of guessing. + */ +function descendantText(components: ScreenComponent[], owner: ScreenComponent): string | undefined { + if (!owner.nodeId) return; + const inside = new Set([owner.nodeId]); + const found = new Set<string>(); + // One ordered pass: a child always follows its parent in a tree flattened depth-first. + for (const item of components) { + if (!item.parentId || !inside.has(item.parentId)) continue; + if (item.nodeId) inside.add(item.nodeId); + const text = accessibleName(item) || elementText(item); + if (text) found.add(text); + if (found.size > 1) return; + } + return [...found][0]; +} + +export type ResolvedSelector = { + selector: RecordedSelector; + ambiguous: boolean; + component?: ScreenComponent; + /** How many elements the point sits inside, so the panel knows how far out it can still walk. */ + levels: number; + /** + * How many elements the chosen selector matches: 1 when it is pinned, more when every candidate + * was ambiguous and the best one was kept and flagged. "Ambiguous" on its own is a verdict; the + * count is the thing a user can act on, and it is what the panel shows them. + */ + matches: number; +}; + +const area = ({ bounds }: ScreenComponent) => bounds.width * bounds.height; + +/** + * Whether the tree describes the app at all. + * + * The runner's snapshot is rooted at the application, so the simulator's own chrome — clock, + * Dynamic Island, battery — is simply not in it, and there is no "only the status bar" state left + * to detect. An empty tree still is one: nothing was attached, or the app has no accessible + * content, and either way a recorded step would name nothing. + */ +export type TreeHealth = "ok" | "no-accessibility"; +export function treeHealth(components: ScreenComponent[]): TreeHealth { + return components.length > 0 ? "ok" : "no-accessibility"; +} + +function covers({ bounds }: ScreenComponent, point: ScreenPoint): boolean { + return point.x >= bounds.x && point.x < bounds.x + bounds.width && point.y >= bounds.y && point.y < bounds.y + bounds.height; +} + +/** + * Every element the point sits inside, innermost first. + * + * The tree is the runner's own, so every element in it is one the runner can address — there is no + * second tree of React Native fibers lying over the same pixels to choose between, and no + * simulator chrome to filter out, because the snapshot is rooted at the application. + * + * `componentsAt` answers with one branch — an element plus its own ancestors — which is too narrow + * here, since the point may sit inside siblings' bounds too. The ladder is every hit, ordered by + * area, which is also the order "walk out to the parent" means on screen. + */ +export function hitLadder(components: ScreenComponent[], point: ScreenPoint, screen?: ScreenSize): ScreenComponent[] { + const hits = components.filter(item => covers(item, point) && !isScreenSized(item, screen)) + .sort((a, b) => area(a) - area(b) || b.depth - a.depth); + // The overlay's own pick leads when it is one of them, so the recorder starts on the element the + // mirror highlights. + const picked = componentsAt(components, point, screen)[0]; + return picked && hits.includes(picked) ? [picked, ...hits.filter(item => item !== picked)] : hits; +} + +/** + * Turns a point on the screen into the selector a test should use. + * + * The hit test is the same one the annotation overlay uses, so what the recorder names is what the + * user sees highlighted. Uniqueness is checked against the whole tree: a selector that would match + * two elements is not pinned, it is a flake, so the next rank is tried. If every rank is ambiguous + * the best one is kept and flagged rather than silently emitting a selector that may hit the wrong + * element. + */ +export function resolveSelector(components: ScreenComponent[], point: ScreenPoint, screen?: ScreenSize, level = 0): ResolvedSelector { + // The innermost element under the point often carries nothing addressable — an unlabelled view, + // a bare container — while something just outside it holds the name or the testID. Reading the + // whole ladder names the thing the user meant rather than giving up on a coordinate. `level` is + // the user walking outwards: it drops that many innermost elements. + const ladder = hitLadder(components, point, screen); + const levels = ladder.length; + const chain = ladder.slice(Math.max(0, Math.min(level, levels - 1))); + const component = chain[0]; + if (!component) return { selector: { kind: "point", point }, ambiguous: false, levels, matches: 1 }; + // A coordinate is what is left when nothing in the chain can be named — the one fallback a point + // has and an element does not, which is why it lives here rather than in `resolveChain`. + return resolveChain(components, chain, levels) + ?? { selector: { kind: "point", point }, ambiguous: false, component, levels, matches: 1 }; +} + +/** + * The element and its ancestors, innermost first: the element-shaped equivalent of the hit ladder. + * + * The point path reaches a chain by hit-testing; an agent that already named an element reaches + * the same shape by walking `parentId` up. Both then go through `resolveChain`, so a step an agent + * records and a step a tap records are resolved by one piece of code rather than two that agree + * for a while. + */ +export function ancestorChain(components: ScreenComponent[], component: ScreenComponent): ScreenComponent[] { + const byId = new Map(components.flatMap(item => item.nodeId ? [[item.nodeId, item] as const] : [])); + const chain = [component]; + const seen = new Set(component.nodeId ? [component.nodeId] : []); + for (let parent = component.parentId; parent !== undefined; ) { + const next = byId.get(parent); + // A cycle cannot happen in a tree and would hang the walk if it ever did. + if (!next || seen.has(parent)) break; + seen.add(parent); + chain.push(next); + parent = next.parentId; + } + return chain; +} + +/** + * How a test should name one element, or nothing when the screen offers no way to name it. + * + * This is the whole of the ranking — depth first, rank within a depth, the first candidate that + * resolves to exactly one element winning whatever its rank, and a position among the matches as + * the last resort. It takes a chain rather than a point because the two callers start from + * different things: a tap starts from a coordinate and an agent starts from an element it read. + */ +export function resolveComponentSelector(components: ScreenComponent[], component: ScreenComponent, screen?: ScreenSize): ResolvedSelector | undefined { + // A root spanning the whole screen names nothing; it is the screen. The point path never reaches + // one because `hitLadder` drops it, and without the same rule here every unnameable element on + // an app whose root carries a title resolves to that title — `screen.getByText("expoapp57")`, + // reported as unique, for a scroll bar. + const chain = ancestorChain(components, component).filter(item => !isScreenSized(item, screen)); + return chain.length > 0 ? resolveChain(components, chain, chain.length) : undefined; +} + +function isScreenSized({ bounds }: ScreenComponent, screen?: ScreenSize): boolean { + return screen !== undefined && bounds.width >= screen.width * .95 && bounds.height >= screen.height * .95; +} + +/** + * The ranks, tried over one chain of elements, innermost first. + * + * Returns nothing when no element in the chain carries anything a query could name. The point path + * answers that with a coordinate; the agent path answers it by saying so, because an agent can + * describe what it cannot name and a coordinate recorded from a `ref` would be a worse step than + * the `agent.act` the agent would write instead. + */ +export function resolveChain(components: ScreenComponent[], chain: ScreenComponent[], levels: number): ResolvedSelector | undefined { + // Only the named ranks; the point fallback is not a candidate, it is what is left when these fail. + // Depth first, rank within a depth: the innermost element that can be named at all wins, because + // that is the thing the user pointed at. An enclosing element is reached by pointing at it, or + // by walking out — not by quietly outranking what is under the pointer. + type Candidate = { selector: NamedSelector; owner: ScreenComponent }; + const candidates: Candidate[] = []; + for (const owner of chain.slice(0, 8)) { + // The role is the runner's own, passed through untouched. Translating it here is what this + // path exists to stop: the engine already decided that a tab bar's button is a `tab`. + const role = QUERYABLE_ROLES.has(owner.role ?? "") ? owner.role! : ""; + const name = accessibleName(owner); + const placeholder = normalize(owner.placeholder); + const text = elementText(owner); + if (owner.identifier) candidates.push({ owner, selector: { kind: "testId", testId: owner.identifier } }); + // A role query with no name matches every element of that role, so the pair is the candidate + // or there is none — which is exactly the search field whose only text is its placeholder. + if (role && name) candidates.push({ owner, selector: { kind: "role", role, name } }); + if (placeholder) candidates.push({ owner, selector: { kind: "placeholder", placeholder } }); + // `getByText` searches the name and the element's own text, innermost match only. + if (name || text) candidates.push({ owner, selector: { kind: "text", text: name || text } }); + if (name) candidates.push({ owner, selector: { kind: "label", label: name } }); + // An icon-only control carries nothing itself: a tab bar item is an unlabelled container with + // the word "Explore" in a sibling of the icon, below the point that was tapped. The one text + // inside it names it, and tapping that text lands inside the same control. + if (!owner.identifier && !name) { + const inside = descendantText(components, owner); + if (inside) candidates.push({ owner, selector: { kind: "label", label: inside } }); + } + } + + // The first candidate that resolves to exactly one element wins, whatever its rank: an ambiguous + // preferred candidate never beats a unique later one, for this element or an enclosing one. + for (const { selector, owner } of candidates) { + if (selectorMatches(components, selector).length === 1) { + return { selector, ambiguous: false, component: owner, levels, matches: 1 }; + } + } + // Nothing was unique, so the element is narrowed by position instead. A bare selector matching + // two is a step that fails with `LOCATOR_AMBIGUOUS`; the same selector at `.nth(i)` is a step + // that runs, and `i` is where the element the user pointed at actually sits among the matches. + for (const { selector, owner } of candidates) { + const found = selectorMatches(components, selector); + const index = found.indexOf(owner); + if (index < 0 || found.length < 2) continue; + return { + selector: { kind: "nth", inner: selector, index, matches: found.length }, + ambiguous: true, component: owner, levels, matches: found.length, + }; + } + // The element matched a candidate the runner would not count it among — a container the + // innermost-only rule drops, say. The selector still describes it better than a coordinate, so + // it is kept and flagged; with no candidate at all, there is nothing here to name. + const fallback = candidates[0]; + return fallback + ? { selector: fallback.selector, ambiguous: true, component: fallback.owner, levels, matches: selectorMatches(components, fallback.selector).length } + : undefined; +} + +/** The elements one selector matches, in the runner's own order. See the `matches*` helpers. */ +function selectorMatches(components: ScreenComponent[], selector: RecordedSelector): ScreenComponent[] { + switch (selector.kind) { + case "testId": return matchesTestId(components, selector.testId); + case "role": return matchesRole(components, selector.role, selector.name); + case "placeholder": return matchesPlaceholder(components, selector.placeholder); + case "label": return matchesLabel(components, selector.label); + case "text": return matchesText(components, selector.text); + // A point names no element, and an `nth` has already been narrowed to one. + default: return []; + } +} + +/** + * Something on this screen a test can wait for, nearest the point first. + * + * `screen.tapAt` is the one step with nothing to wait on: a locator action waits up to the action + * timeout for its element, but a coordinate is always "ready", so a recorded coordinate tap fires + * into a splash screen and misses. This picks a neighbour that does resolve uniquely, which the + * emitter puts in front of the tap as a gate. + */ +export function anchorSelector(components: ScreenComponent[], point: ScreenPoint, screen?: ScreenSize): RecordedSelector | undefined { + const distance = (item: ScreenComponent) => + Math.hypot(item.bounds.x + item.bounds.width / 2 - point.x, item.bounds.y + item.bounds.height / 2 - point.y); + // The screen's own root is the nearest thing to any point and names nothing: gating a step on + // `expect(screen.getByText("<the app>")).toBeVisible()` waits for the application to exist, + // which it did before the test started. + for (const item of components.filter(entry => !isScreenSized(entry, screen)) + .sort((a, b) => distance(a) - distance(b)).slice(0, 40)) { + if (item.identifier && matchesTestId(components, item.identifier).length === 1) return { kind: "testId", testId: item.identifier }; + const placeholder = normalize(item.placeholder); + if (placeholder && matchesPlaceholder(components, placeholder).length === 1) return { kind: "placeholder", placeholder }; + const text = accessibleName(item) || elementText(item); + if (!text) continue; + if (matchesText(components, text).length === 1) return { kind: "text", text }; + if (matchesLabel(components, text).length === 1) return { kind: "label", label: text }; + } + return undefined; +} + +const quote = (value: string) => JSON.stringify(value); + +/** A key that can be written bare in an object literal, as opposed to one that has to be quoted. */ +const BARE_KEY = /^[A-Za-z_$][A-Za-z0-9_$]*$/; + +/** + * A JSON number, exactly. Deliberately not `Number(value)`, which also accepts `0x10`, `1_000`, + * `Infinity` and a leading `+` — each of which would reach the file as a different number than the + * text the user typed, or as an identifier. + */ +const JSON_NUMBER = /^-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?$/; + +/** + * One param value as a literal: a number, `true`, `false` or `null` unquoted, anything else as a + * string. + * + * The editor has no type picker, so the text is the type: a user typing `3` into a field called + * `attempts` means the number, and one typing `a@b.com` means the string. Surrounding whitespace + * is not part of a number or a keyword, so it is dropped when deciding and when writing one; a + * string keeps every character it was given, since a trailing space may be exactly what the + * instruction is about. + */ +function paramValueSource(value: string): string { + const trimmed = value.trim(); + if (trimmed === "true" || trimmed === "false" || trimmed === "null") return trimmed; + if (JSON_NUMBER.test(trimmed)) return trimmed; + return quote(value); +} + +/** + * The `{ params: … }` options argument, or nothing at all. + * + * Nothing at all is the point: `params` is the only `ActOptions` key the panel records, so with no + * params there is no options object to write, and `agent.act("…")` stays the line it always was. + * A row with a blank key is not a param — it is a row the user has started and not finished — so + * it is dropped rather than emitted as `"": …`. + */ +function paramsSource(params?: AgentParams): string { + const entries = Object.entries(params ?? {}).filter(([key]) => key.trim() !== ""); + if (entries.length === 0) return ""; + const body = entries.map(([key, value]) => + `${BARE_KEY.test(key) ? key : quote(key)}: ${paramValueSource(value)}`).join(", "); + return `, { params: { ${body} } }`; +} + +/** The locator expression a step emits, e.g. `screen.getByRole("tab", "Explore")`. */ +export function selectorSource(selector: RecordedSelector): string { + switch (selector.kind) { + case "testId": return `screen.getByTestId(${quote(selector.testId)})`; + case "role": return `screen.getByRole(${quote(selector.role)}, ${quote(selector.name)})`; + case "placeholder": return `screen.getByPlaceholder(${quote(selector.placeholder)})`; + case "text": return `screen.getByText(${quote(selector.text)})`; + case "label": return `screen.getByLabel(${quote(selector.label)})`; + case "point": return `screen.tapAt({ x: ${Math.round(selector.point.x)}, y: ${Math.round(selector.point.y)} })`; + // Chained on the locator, as the runner's own narrowing is. + case "nth": return `${selectorSource(selector.inner)}.nth(${selector.index})`; + } +} + +/** + * The expression a step emits, without the `await` or the semicolon the file needs. + * + * The element menu keeps this on each action as a tooltip, while its compact row uses the action's + * plain-language label. The emitter wraps it; nothing else formats steps. + */ +export function stepExpression(step: + | { kind: PinnedStepKind; selector: RecordedSelector; value?: string; direction?: SwipeDirection; timeoutMs?: number } + | { kind: "agentAct"; instruction: string; params?: AgentParams } + | { kind: "agentAssert"; instruction: string; vision?: VisionMode } + | { kind: "agentWaitFor"; instruction: string; vision?: VisionMode } +): string { + // Neither agent call takes a locator, so neither reads a selector: the instruction is the step. + // `act` has no `vision` at all — the act model pulls pixels itself when the tree lacks what it + // needs — and `assert`'s defaults to `false`, so an options object is emitted only when the user + // asked for something other than the default. + if (step.kind === "agentAct") return `agent.act(${quote(step.instruction)}${paramsSource(step.params)})`; + if (step.kind === "agentAssert" || step.kind === "agentWaitFor") { + // `waitFor` judges the same way `assert` does, on a loop, so it takes the same `vision` and + // leaves it out for the same reason: `false` is what the runner does anyway. + const call = step.kind === "agentAssert" ? "assert" : "waitFor"; + const vision = step.vision === "only" ? quote("only") : step.vision === true ? "true" : undefined; + return vision + ? `agent.${call}(${quote(step.instruction)}, { vision: ${vision} })` + : `agent.${call}(${quote(step.instruction)})`; + } + const locator = selectorSource(step.selector); + const timeout = step.timeoutMs === undefined ? "" : `{ timeout: ${step.timeoutMs} }`; + switch (step.kind) { + // A point is already a whole action — `screen.tapAt(...)` — so there is nothing to call on it. + case "tap": return step.selector.kind === "point" ? locator : `${locator}.tap()`; + case "doubleTap": return `${locator}.doubleTap()`; + case "longPress": return `${locator}.longPress()`; + case "fill": return `${locator}.fill(${quote(step.value ?? "")})`; + case "scrollUntilVisible": return `screen.scrollUntilVisible(${locator})`; + case "assertVisible": return `expect(${locator}).toBeVisible(${timeout})`; + // The runner's own negation, which waits for the element to stay gone rather than catching it + // mid-transition. `toBeHidden` is the other spelling and means something else on a device: + // an element that has gone is absent from the tree, not hidden in it. + case "assertNotVisible": return `expect(${locator}).not.toBeVisible(${timeout})`; + case "assertText": return `expect(${locator}).toHaveText(${quote(step.value ?? "")}${timeout ? `, ${timeout}` : ""})`; + case "assertContainsText": return `expect(${locator}).toContainText(${quote(step.value ?? "")}${timeout ? `, ${timeout}` : ""})`; + case "assertValue": return `expect(${locator}).toHaveValue(${quote(step.value ?? "")}${timeout ? `, ${timeout}` : ""})`; + case "assertEnabled": return `expect(${locator}).toBeEnabled(${timeout})`; + case "assertDisabled": return `expect(${locator}).toBeDisabled(${timeout})`; + case "assertSelected": return `expect(${locator}).toBeSelected(${timeout})`; + // The one command with two forms. A point names no element, so there is nothing to scope the + // gesture to and the viewport is swiped; a selector scopes it to the element it names, which + // is the difference between flicking the screen and flicking one carousel on it. `momentum` + // is left at its default, so `direction` is the whole of the options object. + case "swipe": return `${step.selector.kind === "point" ? "screen" : locator}.swipe({ direction: ${quote(step.direction ?? "up")} })`; + } +} + +/** The verb each kind reads as in the panel's step list. */ +const STEP_VERB: Record<RecordedStepKind, string> = { + tap: "Tap", + doubleTap: "Double tap", + longPress: "Long press", + fill: "Type into", + scrollUntilVisible: "Scroll to", + assertVisible: "Expect visible", + assertNotVisible: "Expect not visible", + assertText: "Expect text of", + assertContainsText: "Expect text in", + assertValue: "Expect value of", + assertEnabled: "Expect enabled", + assertDisabled: "Expect disabled", + assertSelected: "Expect selected", + swipe: "Swipe", + // An agent step reads as what the agent was told, so the verb says who is acting and the + // instruction says the rest. + agentAct: "Agent", + agentAssert: "Agent check", + agentWaitFor: "Agent wait", +}; + +/** Whether the step carries text the user has to supply: one types it, the rest compare with it. */ +export function stepNeedsValue(kind: RecordedStepKind): boolean { + return kind === "fill" || kind === "assertText" || kind === "assertContainsText" || kind === "assertValue"; +} + +/** How an agent step reads in the panel's step list: the verb, then the sentence it was given. */ +export function agentStepLabel(kind: AgentStepKind, instruction: string): string { + return `${STEP_VERB[kind]} ${JSON.stringify(instruction)}`; +} + +/** How a step reads in the panel's step list. */ +export function stepLabel(kind: PinnedStepKind, selector: RecordedSelector, value?: string): string { + // Which way it went is the step, so it belongs in the verb rather than trailing the element. + if (kind === "swipe") { + const way = `Swipe ${value ?? "up"}`; + return selector.kind === "point" ? `${way} (viewport)` : `${way} on ${stepLabel("tap", selector).replace(/^Tap /, "")}`; + } + const verb = STEP_VERB[kind]; + const suffix = stepNeedsValue(kind) ? ` → ${JSON.stringify(value ?? "")}` : ""; + switch (selector.kind) { + case "testId": return `${verb} #${selector.testId}${suffix}`; + case "role": return `${verb} "${selector.name}" (${selector.role})${suffix}`; + case "placeholder": return `${verb} "${selector.placeholder}"${suffix}`; + case "text": return `${verb} "${selector.text}"${suffix}`; + case "label": return `${verb} "${selector.label}"${suffix}`; + case "point": return `${verb} at ${Math.round(selector.point.x)}, ${Math.round(selector.point.y)}`; + // The position is the whole point of the step, so the list says which of the matches it is. + case "nth": return `${stepLabel(kind, selector.inner, value)} (${selector.index + 1} of ${selector.matches})`; + } +} + +/** + * One row of the element menu: a step the element supports and the code it would add. + * + * `needsValue` rows carry text the user has to supply — `fill` types it, `toHaveText` compares it — + * so the menu asks before the step is recorded rather than emitting an empty string literal. + */ +export type StepOffer = { + /** + * Pinned only. An agentic step is never a row here: it names no element, so it cannot be gated + * on what one carries, and converting a recorded step to one would throw away the locator that + * step exists for. The mirror offers Agentic from a branch of its own. + */ + kind: PinnedStepKind; + /** The code the row exposes as a tooltip, e.g. `screen.getByText("Explore").tap()`. */ + source: string; + needsValue?: boolean; + /** What a `needsValue` row starts from: the field's current value, or the element's own text. */ + suggestedValue?: string; +}; + +/** The roles whose elements take typed text, as `e2e`'s `Role` union names them. */ +const TEXT_INPUT_ROLES = new Set(["textbox", "searchbox", "combobox", "spinbutton"]); + +/** + * What the panel does to the device when a row is chosen. + * + * The menu is how a step is recorded, and a step that is recorded but never taken leaves the flow + * on the wrong screen — the next row would be resolved against a screen the test will not be on. + * So a row that acts, acts. `none` is for the rows that genuinely change nothing: an assertion + * reads the screen, and `scrollUntilVisible` is only ever offered for an element the user just + * pointed at, which is on screen already, so its search loop has nothing to do here. + */ +export type StepEffect = "tap" | "doubleTap" | "type" | "swipe" | "none"; + +export function stepEffect(kind: RecordedStepKind): StepEffect { + switch (kind) { + case "tap": return "tap"; + // Two taps, which is the closest the panel's one-shot tap can come to the gesture the step + // emits. It is what keeps the app on the screen the next step is resolved against; whether + // the device coalesces them into a real double tap is the app's business, not the step's. + case "doubleTap": return "doubleTap"; + case "fill": return "type"; + // A swipe moves the screen under the rest of the flow, so it is taken as well as recorded. + case "swipe": return "swipe"; + // `longPress` has no equivalent in the panel's input path, so the step is recorded and the app + // stays where it is — the same as every assertion, which reads the screen and changes nothing. + default: return "none"; + } +} + +/** + * What the runner can do with this element, as menu rows. + * + * Derived from the element's role and what it carries, never from the app: a role in + * `TEXT_INPUT_ROLES`, or a placeholder — which only a text field has — is what makes a row a + * `fill`, and an element with its own text is what makes `toHaveText` a comparison with something + * rather than with nothing. + * + * A coordinate gets the tap alone. There is no element behind it to be visible, to hold text or to + * be scrolled to, which is the same refusal the recorder makes for a point assertion. + */ +export function stepOffers(selector: RecordedSelector, component?: StepTarget): StepOffer[] { + // A narrowed selector supports whatever its inner one does: the narrowing is on the locator, and + // every action and matcher chains after it. + const offer = (kind: PinnedStepKind, value?: string, needsValue?: boolean): StepOffer => ({ + kind, + source: stepExpression({ kind, selector, ...(value === undefined ? {} : { value }) }), + ...(needsValue ? { needsValue: true } : {}), + ...(value === undefined ? {} : { suggestedValue: value }), + }); + if (selector.kind === "point") return [offer("tap")]; + const role = component?.role ?? ""; + const placeholder = normalize(component?.placeholder); + const typeable = TEXT_INPUT_ROLES.has(role) || placeholder !== ""; + // One row per kind, not one per variant. Enumerating `longPress` beside `tap` and `toHaveText` + // beside `toBeVisible` made the menu a list of near-duplicates of the same selector, which is + // harder to read than the three choices it is actually offering. + return [ + offer("tap"), + ...(typeable ? [offer("fill", normalize(component?.value), true)] : []), + offer("assertVisible"), + offer("scrollUntilVisible"), + ]; +} + +/** + * Every step the runner can express for this element, for the menu's "all commands" list. + * + * `stepOffers` is the short list — the four things most steps are — and it stays short on purpose. + * This is the rest of what the runner has, which is the difference between a recorder that can + * assert an element is *gone* and one whose user has to leave the panel and write that line by + * hand. Each row is still gated on what the element carries: only a text field can be filled or + * have a value, and only an element with text of its own can be compared against text. + * + * A coordinate still gets the tap alone: there is no element behind it to be visible, to hold + * text, to be enabled or to be scrolled to. + */ +export function allStepOffers(selector: RecordedSelector, component?: StepTarget): StepOffer[] { + const offer = (kind: PinnedStepKind, value?: string): StepOffer => ({ + kind, + source: stepExpression({ kind, selector, ...(value === undefined ? {} : { value }) }), + ...(stepNeedsValue(kind) ? { needsValue: true } : {}), + ...(value === undefined ? {} : { suggestedValue: value }), + }); + // One row per direction, carrying its own exact code; the menu gathers them into one branch. + // The direction travels in `suggestedValue` because it is what the chosen row stands for, not + // text the row asks for — `stepNeedsValue("swipe")` is false, so no form opens. + const swipes = SWIPE_DIRECTIONS.map(direction => ({ + kind: "swipe" as const, + source: stepExpression({ kind: "swipe", selector, direction }), + suggestedValue: direction, + })); + // A coordinate still has no element to assert about, fill or scroll to — but it does have a + // viewport, and `screen.swipe` is about the viewport rather than about anything on it. + if (selector.kind === "point") return [offer("tap"), ...swipes]; + const role = component?.role ?? ""; + const placeholder = normalize(component?.placeholder); + const typeable = TEXT_INPUT_ROLES.has(role) || placeholder !== ""; + const text = normalize(component?.label) || normalize(component?.text); + return [ + offer("tap"), + offer("doubleTap"), + offer("longPress"), + ...(typeable ? [offer("fill", normalize(component?.value))] : []), + ...swipes, + offer("scrollUntilVisible"), + offer("assertVisible"), + offer("assertNotVisible"), + ...(text ? [offer("assertText", text), offer("assertContainsText", text)] : []), + ...(typeable ? [offer("assertValue", normalize(component?.value))] : []), + offer("assertEnabled"), + offer("assertDisabled"), + offer("assertSelected"), + ]; +} + +/** Whether a command acts on the app or only reads it, for the menu's two groups. */ +export function stepIsAssertion(kind: RecordedStepKind): boolean { + // `agentAssert` is judged by a model rather than matched by a locator, and it still reads the + // screen without changing it, which is the only distinction this answers. + return kind.startsWith("assert") || kind === "agentAssert"; +} + +/** A test title turned into a file name, e.g. `the tab bar works` → `the-tab-bar-works`. */ +export function testFileSlug(title: string): string { + const slug = title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60); + return slug || "recorded"; +} + +/** One emitted line, and whether the runner will report a step for it. */ +type EmittedLine = { text: string; call?: true }; + +function stepSource(step: RecordedStep): EmittedLine[] { + // An agent step names no element: there is no selector to be ambiguous, no `nth` to explain and + // no coordinate to gate behind an anchor, so the instruction is the whole of it. + if (isAgentStep(step)) return [{ text: ` await ${stepExpression(step)};`, call: true }]; + const lines: EmittedLine[] = []; + // A positional step runs, which a bare ambiguous one does not, so this is a note and not a TODO: + // nothing has to be edited for the test to work, and the durable fix is still named. + if (step.selector.kind === "nth") { + lines.push({ text: ` // ${step.selector.matches} elements match this selector; this is the one that was recorded. A testID would pin it by name.` }); + } else if (step.ambiguous) { + lines.push({ text: ` // TODO: this matched more than one element. Add a testID to pin it.` }); + } + if (step.selector.kind === "point") { + // A swipe of the viewport is the one step a point *means*: `screen.swipe({ direction })` names + // no coordinate, so nothing in it can break on a layout change and the TODO was simply untrue + // of it. The anchor below still applies — a swipe is as actionable as a tap and will fire into + // a splash screen just the same. + if (step.kind !== "swipe") { + lines.push({ text: ` // TODO: nothing resolved here. A coordinate tap breaks on any layout change.` }); + } + // A locator action waits for its element; a coordinate does not, so it would fire into a + // splash screen. The anchor is the wait the tap itself cannot express — and the runner reports + // it as a step of its own, which is why it counts as a call. + if (step.anchor && step.anchor.kind !== "point") { + lines.push({ text: ` await expect(${selectorSource(step.anchor)}).toBeVisible();`, call: true }); + } + // A tap at the coordinate, and a swipe of the viewport the coordinate is in: neither needs an + // element, which is the whole of what a point lacks. + if (step.kind === "tap" || step.kind === "swipe") { lines.push({ text: ` await ${stepExpression(step)};`, call: true }); return lines; } + // There is nothing to assert about, type into, or scroll to a bare coordinate. + lines.push({ text: ` // Skipped a step on a point with no element.` }); + return lines; + } + lines.push({ text: ` await ${stepExpression(step)};`, call: true }); + return lines; +} + +/** + * Where one step's lines sit in the emitted file, as 1-based line numbers. + * + * `calls` are the lines the runner will report a step for; the rest are the TODO comments a step + * carries with it. A run marks the calls and leaves the comments alone. + */ +export type EmittedStepSpan = { stepId: string; from: number; to: number; calls: number[] }; +export type EmittedTest = { + source: string; + spans: EmittedStepSpan[]; + /** + * Every awaited line in file order, `await app.open()` first. + * + * This is what maps the runner's live progress to a line. A `step` event carries + * `identity.stepIndex` and no source position — only the finished report has one — and the runner + * reports one step per awaited call, in order, so the index *is* the position in this list. + * The report reconciles it at the end, so a mismatch corrects itself rather than persisting. + */ + calls: number[]; +}; + +/** + * Emits the whole test from the steps, and says which lines each step produced. + * + * It is a full re-emit every time, so deleting a step or renaming the test just re-renders — + * nothing is ever patched in place, and the preview and the saved file are always the same text. + * + * The spans are what lets a run report land on the right line. A step is not one line: an + * ambiguous one carries a TODO above its call, and a coordinate tap carries a TODO and an anchor + * gate as well. Recovering that by diffing emitted sources, as the preview's new-line animation + * does, is good enough to flash a line and far too weak to attribute a pass or a failure, so the + * emitter reports it instead of the reader guessing. + * + * A recording is pinned locators by default — that is what makes a recorded test repeatable — so + * the fixtures and the header comment follow the steps: `agent` is destructured, and the comment + * says what the file is, only once a recording actually contains an agentic step. A recording + * without one emits exactly what it always did. + */ +export function emitE2eTestLines(steps: readonly RecordedStep[], options: { title: string }): EmittedTest { + const agentic = steps.some(isAgentStep); + const lines: EmittedLine[] = [ + { text: `import { test } from "@e2e-dev/mobile";` }, + { text: `import { expect } from "e2e";` }, + { text: `` }, + { text: agentic + ? `// Recorded from the Testing panel. Pinned locators where an element was named, and agent steps where it was described.` + : `// Recorded from the Testing panel. Every step is a pinned locator, driven by no agent.` }, + { text: `test(${quote(options.title)}, async ({ ${agentic ? "agent, app, screen" : "app, screen"} }) => {` }, + // The runner reports `app.open` as a step like any other, so it counts as a call. + { text: ` await app.open();`, call: true }, + { text: `` }, + ]; + const spans: EmittedStepSpan[] = []; + if (steps.length === 0) lines.push({ text: " // Tap the mirror to record steps." }); + for (const step of steps) { + const from = lines.length + 1; + const emitted = stepSource(step); + lines.push(...emitted); + spans.push({ + stepId: step.id, from, to: lines.length, + calls: emitted.flatMap((line, index) => line.call ? [from + index] : []), + }); + } + lines.push({ text: `});` }, { text: `` }); + return { + source: lines.map(line => line.text).join("\n"), + spans, + calls: lines.flatMap((line, index) => line.call ? [index + 1] : []), + }; +} + +export function emitE2eTest(steps: readonly RecordedStep[], options: { title: string }): string { + return emitE2eTestLines(steps, options).source; +} + +/** + * The sentence an `agent.act` row starts from, written out of the element the user right-clicked. + * + * It is a typing shortcut and nothing else: the element is not stored on the step and never + * reaches the file, because the runner resolves the instruction against the screen it finds at run + * time. Starting from `tap the "Explore" tab` is the difference between a row the user edits and a + * blank box they have to compose in, and both end up as the same free text. + * + * Empty when the element carries no name at all — there is nothing to call it, so the user says + * what they mean rather than being handed a sentence about nothing. + */ +export function agentActSuggestion(target?: StepTarget): string { + const name = normalize(target?.label) || normalize(target?.text) || normalize(target?.placeholder); + if (!name) return ""; + const role = QUERYABLE_ROLES.has(target?.role ?? "") ? target!.role! : ""; + return role ? `tap the ${JSON.stringify(name)} ${role}` : `tap ${JSON.stringify(name)}`; +} diff --git a/packages/shared/test/test-recording.test.ts b/packages/shared/test/test-recording.test.ts new file mode 100644 index 00000000..a3f60e8f --- /dev/null +++ b/packages/shared/test/test-recording.test.ts @@ -0,0 +1,801 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { ScreenComponent } from "../src/screen-annotations.ts"; +import { scaledComponents } from "../src/screen-annotations.ts"; +import { agentActSuggestion, agentStepLabel, allStepOffers, ancestorChain, anchorSelector, emitE2eTest, emitE2eTestLines, treeHealth, resolveComponentSelector, resolveSelector, selectorSource, stepLabel, stepOffers, swipeDirection, swipePath, testFileSlug } from "../src/test-recording.ts"; +import type { RecordedSelector, RecordedStep, VisionMode } from "../src/test-recording.ts"; + +/** + * One element as the recorder reads it: the e2e runner's own projected node, where `role` is the + * runner's role, `label` its accessible *name*, and `text` the element's own text. Every fixture + * below was captured from a running simulator through the same path the recorder uses, so a test + * that passes here is a locator the runner resolves the same way. + */ +function component(fields: Partial<ScreenComponent> & { bounds: ScreenComponent["bounds"] }): ScreenComponent { + return { name: fields.label ?? "Element", depth: 1, source: "accessibility", nodeId: fields.identifier ?? fields.label ?? "node", ...fields }; +} + +const TAB = { x: 0, y: 800, width: 100, height: 40 }; +const SCREEN = { width: 402, height: 874 }; + +test("a testID wins over the role and the text", () => { + const components = [component({ bounds: TAB, identifier: "ExploreTab", role: "button", label: "Explore", text: "Explore" })]; + const { selector, ambiguous } = resolveSelector(components, { x: 50, y: 820 }); + assert.deepEqual(selector, { kind: "testId", testId: "ExploreTab" }); + assert.equal(ambiguous, false); + assert.equal(selectorSource(selector), 'screen.getByTestId("ExploreTab")'); +}); + +test("without a testID the role and the accessible name are used", () => { + const components = [component({ bounds: TAB, role: "button", label: "Explore" })]; + const { selector } = resolveSelector(components, { x: 50, y: 820 }); + assert.deepEqual(selector, { kind: "role", role: "button", name: "Explore" }); + assert.equal(selectorSource(selector), 'screen.getByRole("button", "Explore")'); +}); + +test("the runner's role is emitted verbatim, with no table in between", () => { + // The engine already mapped the platform's element type onto its role vocabulary when it + // projected the snapshot, and it is the only thing that gets a say: a second mapping here is + // what made `getByRole("link", …)` and `getByRole("rcttext", …)` reach the file. + for (const role of ["tab", "textbox", "listitem", "navigation", "progressbar", "switch", "heading"]) { + const components = [component({ bounds: TAB, role, label: "Thing" })]; + assert.deepEqual(resolveSelector(components, { x: 50, y: 820 }).selector, { kind: "role", role, name: "Thing" }); + } +}); + +test("a role getByRole does not accept falls through to the text rank, never renamed", () => { + // The engine keeps an element type it has no role for as the type itself (`other`, + // `application`), and `getByRole`'s union is closed, so querying one would not compile. The + // element is named by what it says instead — it is never quietly called something else. + for (const role of ["other", "text", "application", "web-view"]) { + const components = [component({ bounds: TAB, role, label: "Thing", text: "Thing" })]; + assert.deepEqual(resolveSelector(components, { x: 50, y: 820 }).selector, { kind: "text", text: "Thing" }); + } +}); + +test("a selector matching two elements is narrowed to the one that was pointed at", () => { + // Two buttons named "Open": the role selector matches both, and so does every other candidate. + // A bare selector here is a step that fails with LOCATOR_AMBIGUOUS, so it is narrowed by + // position instead — the first one, because that is the one under the pointer. + const components = [ + component({ bounds: TAB, role: "button", label: "Open", nodeId: "a" }), + component({ bounds: { x: 0, y: 900, width: 100, height: 40 }, role: "button", label: "Open", nodeId: "b" }), + ]; + const { selector, ambiguous, matches } = resolveSelector(components, { x: 50, y: 820 }); + assert.deepEqual(selector, { kind: "nth", inner: { kind: "role", role: "button", name: "Open" }, index: 0, matches: 2 }); + assert.equal(ambiguous, true, "positional is still less durable than a name, and the panel says so"); + assert.equal(matches, 2); + assert.equal(selectorSource(selector), 'screen.getByRole("button", "Open").nth(0)'); +}); + +test("the index is the pointed-at element's own, not the first match", () => { + // The element under the pointer is the *second* match. Assuming the first would record a step + // that runs and does the wrong thing, which is worse than one that fails. + const components = [ + component({ bounds: TAB, role: "button", label: "Open", nodeId: "a" }), + component({ bounds: { x: 0, y: 900, width: 100, height: 40 }, role: "button", label: "Open", nodeId: "b" }), + ]; + const { selector } = resolveSelector(components, { x: 50, y: 920 }); + assert.deepEqual(selector, { kind: "nth", inner: { kind: "role", role: "button", name: "Open" }, index: 1, matches: 2 }); +}); + +test("a tap on nothing records a coordinate", () => { + const { selector, ambiguous } = resolveSelector([], { x: 12.4, y: 20.6 }); + assert.deepEqual(selector, { kind: "point", point: { x: 12.4, y: 20.6 } }); + assert.equal(ambiguous, false); + assert.equal(selectorSource(selector), "screen.tapAt({ x: 12, y: 21 })"); +}); + +test("a hidden node is not a match for a role query", () => { + // The runner skips `visibleToUser: false` for a role query, so counting it here would flag a + // step the runner resolves uniquely. + const components = [ + component({ bounds: TAB, role: "button", label: "Open", nodeId: "shown" }), + component({ bounds: { x: 0, y: 900, width: 100, height: 40 }, role: "button", label: "Open", nodeId: "gone", hidden: true }), + ]; + const { selector, ambiguous } = resolveSelector(components, { x: 50, y: 820 }); + assert.deepEqual(selector, { kind: "role", role: "button", name: "Open" }); + assert.equal(ambiguous, false); +}); + +// A search field on the explore-list screen, exactly as the runner reports it: a `TextField` +// projected to `textbox`, carrying its placeholder and **no name at all**. +const SEARCH_FIELD = component({ + bounds: { x: 24, y: 476, width: 354, height: 35 }, + role: "textbox", placeholder: "Search transactions", value: "Search transactions", nodeId: "search", +}); + +test("a field whose only text is its placeholder is named by the placeholder", () => { + // Reading iOS accessibility offered the placeholder as the element's name, so the recorder + // emitted `getByRole("textbox", "Search transactions")` — which matches nothing, because the + // runner reads the name from the label and this field has none. A role query with no name is + // not a candidate at all, and the placeholder is. + const { selector, ambiguous } = resolveSelector([SEARCH_FIELD], { x: 200, y: 490 }); + assert.deepEqual(selector, { kind: "placeholder", placeholder: "Search transactions" }); + assert.equal(ambiguous, false); + assert.equal(selectorSource(selector), 'screen.getByPlaceholder("Search transactions")'); + assert.equal(stepLabel("tap", selector), 'Tap "Search transactions"'); +}); + +test("a placeholder is only reached for when the element has a name of its own", () => { + const named = component({ ...SEARCH_FIELD, role: "textbox", label: "Search", placeholder: "Search transactions" }); + assert.deepEqual(resolveSelector([named], { x: 200, y: 490 }).selector, { kind: "role", role: "textbox", name: "Search" }); +}); + +test("text echoed by a parent onto its child is one match, not two", () => { + // A device tree echoes text upwards: iOS reports a React Native `Text` host view and the + // `StaticText` inside it with the same label. The runner answers `getByText` with the innermost + // match only, so a recorder that counts both puts a "matched more than one element" TODO on a + // line that works. + const components = [ + component({ bounds: { x: 24, y: 140, width: 354, height: 44 }, role: "text", label: "Explore Cities", text: "Explore Cities", nodeId: "outer" }), + component({ bounds: { x: 24, y: 140, width: 354, height: 44 }, role: "text", label: "Explore Cities", text: "Explore Cities", nodeId: "inner", parentId: "outer" }), + ]; + const { selector, ambiguous } = resolveSelector(components, { x: 100, y: 160 }); + assert.deepEqual(selector, { kind: "text", text: "Explore Cities" }); + assert.equal(ambiguous, false); + assert.equal(selectorSource(selector), 'screen.getByText("Explore Cities")'); +}); + +test("two separate elements with the same text are narrowed, not left ambiguous", () => { + // Innermost-only drops a match that *contains* another. Siblings contain nothing, so both stand + // and the step is narrowed to the one that was pointed at. + const components = [ + component({ bounds: TAB, role: "text", label: "Paris", nodeId: "a" }), + component({ bounds: { x: 0, y: 900, width: 100, height: 40 }, role: "text", label: "Paris", nodeId: "b" }), + ]; + const resolved = resolveSelector(components, { x: 50, y: 820 }); + assert.equal(resolved.ambiguous, true); + assert.equal(resolved.matches, 2); + assert.equal(selectorSource(resolved.selector), 'screen.getByText("Paris").nth(0)'); +}); + +test("the emitted test is deterministic and imports the e2e API", () => { + const source = emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "role", role: "tab", name: "Explore" }, label: "" }, + { id: "2", kind: "assertVisible", selector: { kind: "text", text: "Go To Explore List" }, label: "" }, + { id: "3", kind: "tap", selector: { kind: "placeholder", placeholder: "Search transactions" }, label: "" }, + ], { title: "the explore tab opens" }); + assert.match(source, /import \{ test \} from "@e2e-dev\/mobile";/); + assert.match(source, /test\("the explore tab opens", async \(\{ app, screen \}\) => \{/); + assert.match(source, /await screen\.getByRole\("tab", "Explore"\)\.tap\(\);/); + assert.match(source, /await expect\(screen\.getByText\("Go To Explore List"\)\)\.toBeVisible\(\);/); + assert.match(source, /await screen\.getByPlaceholder\("Search transactions"\)\.tap\(\);/); + assert.doesNotMatch(source, /agent\.act/); +}); + +test("assertion timeouts emit when explicitly selected", () => { + const selector = { kind: "testId" as const, testId: "homeScreenFeedTabs-selector-0" }; + const source = emitE2eTest([ + { id: "1", kind: "assertVisible", selector, label: "", timeoutMs: 30_000 }, + { id: "2", kind: "assertNotVisible", selector, label: "", timeoutMs: 5_000 }, + { id: "3", kind: "assertText", selector, label: "", value: "Ready", timeoutMs: 12_000 }, + { id: "4", kind: "assertEnabled", selector, label: "" }, + ], { title: "timeouts" }); + assert.match(source, /toBeVisible\(\{ timeout: 30000 \}\)/); + assert.match(source, /not\.toBeVisible\(\{ timeout: 5000 \}\)/); + assert.match(source, /toHaveText\("Ready", \{ timeout: 12000 \}\)/); + assert.match(source, /toBeEnabled\(\)/); +}); + +test("an unreliable step carries a TODO into the file", () => { + const source = emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "role", role: "button", name: "Open" }, label: "", ambiguous: true }, + { id: "2", kind: "tap", selector: { kind: "point", point: { x: 4, y: 8 } }, label: "" }, + ], { title: "t" }); + assert.match(source, /TODO: this matched more than one element/); + assert.match(source, /TODO: nothing resolved here/); + assert.match(source, /await screen\.tapAt\(\{ x: 4, y: 8 \}\);/); +}); + +test("an empty recording still emits a runnable test", () => { + const source = emitE2eTest([], { title: "nothing yet" }); + assert.match(source, /await app\.open\(\);/); + assert.match(source, /Tap the mirror to record steps/); +}); + +test("step labels and file names read back to the user", () => { + assert.equal(stepLabel("tap", { kind: "role", role: "tab", name: "Home" }), 'Tap "Home" (tab)'); + assert.equal(stepLabel("assertVisible", { kind: "testId", testId: "Back" }), "Expect visible #Back"); + assert.equal(testFileSlug("The tab bar works!"), "the-tab-bar-works"); + assert.equal(testFileSlug("***"), "recorded"); +}); + +test("an enclosing element names a tap its own leaf cannot", () => { + // Tapping inside a row: the leaf carries nothing, the row carries the testID. + const components = [ + component({ bounds: TAB, role: "other", identifier: "explore-tab", nodeId: "parent" }), + component({ bounds: { x: 10, y: 810, width: 80, height: 20 }, role: "other", nodeId: "leaf", parentId: "parent" }), + ]; + const { selector, ambiguous, component: owner } = resolveSelector(components, { x: 50, y: 820 }); + assert.deepEqual(selector, { kind: "testId", testId: "explore-tab" }); + assert.equal(ambiguous, false); + // The outline follows the element the selector names, not the one under the pointer. + assert.equal(owner?.nodeId, "parent"); +}); + +test("walking out picks the enclosing element instead of the one under the pointer", () => { + const components = [ + component({ bounds: { x: 0, y: 100, width: 300, height: 80 }, identifier: "card", role: "button", label: "Open city", nodeId: "card" }), + component({ bounds: { x: 10, y: 110, width: 100, height: 20 }, role: "text", label: "Paris", nodeId: "label", parentId: "card" }), + ]; + const inner = resolveSelector(components, { x: 50, y: 120 }); + assert.deepEqual(inner.selector, { kind: "text", text: "Paris" }); + assert.equal(inner.levels, 2); + + const outer = resolveSelector(components, { x: 50, y: 120 }, undefined, 1); + assert.deepEqual(outer.selector, { kind: "testId", testId: "card" }); + // Past the last level the outermost element is kept rather than falling to a coordinate. + assert.deepEqual(resolveSelector(components, { x: 50, y: 120 }, undefined, 9).selector, outer.selector); +}); + +test("a coordinate tap waits for the screen before it fires", () => { + // A locator action waits for its element; `tapAt` does not, so the first step of a recording + // used to fire during the splash. + const components = [component({ bounds: { x: 0, y: 0, width: 200, height: 40 }, role: "text", label: "Welcome" })]; + const anchor = anchorSelector(components, { x: 100, y: 300 }); + assert.deepEqual(anchor, { kind: "text", text: "Welcome" }); + + const source = emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "point", point: { x: 100, y: 300 } }, label: "", anchor }, + ], { title: "t" }); + const gate = source.indexOf('await expect(screen.getByText("Welcome")).toBeVisible();'); + assert.ok(gate > 0, "the gate must be emitted"); + assert.ok(gate < source.indexOf("screen.tapAt"), "the gate must come before the tap it guards"); +}); + +test("an anchor is only offered when it names exactly one element", () => { + const twice = [ + component({ bounds: { x: 0, y: 0, width: 50, height: 20 }, role: "text", label: "Open", nodeId: "a" }), + component({ bounds: { x: 0, y: 40, width: 50, height: 20 }, role: "text", label: "Open", nodeId: "b" }), + ]; + assert.equal(anchorSelector(twice, { x: 500, y: 500 }), undefined); +}); + +test("an icon-only control is named by the one piece of text inside it", () => { + // The container carries nothing, the word is in a sibling of the icon, and the tap landed on + // the icon. Tapping the text lands inside the same control. + const components = [ + component({ bounds: TAB, role: "other", nodeId: "tab" }), + component({ bounds: { x: 30, y: 805, width: 24, height: 24 }, role: "image", nodeId: "icon", parentId: "tab" }), + component({ bounds: { x: 10, y: 830, width: 80, height: 10 }, role: "text", label: "Explore", nodeId: "label", parentId: "tab" }), + ]; + // The point is on the icon, not on the word. + const { selector, ambiguous } = resolveSelector(components, { x: 40, y: 810 }); + assert.deepEqual(selector, { kind: "label", label: "Explore" }); + assert.equal(ambiguous, false); +}); + +test("a control holding two texts is left unnamed rather than guessed at", () => { + const components = [ + component({ bounds: TAB, role: "other", nodeId: "row" }), + component({ bounds: { x: 10, y: 805, width: 40, height: 10 }, role: "text", label: "Paris", nodeId: "a", parentId: "row" }), + component({ bounds: { x: 10, y: 820, width: 40, height: 10 }, role: "text", label: "France", nodeId: "b", parentId: "row" }), + ]; + assert.equal(resolveSelector(components, { x: 70, y: 810 }).selector.kind, "point"); +}); + +test("an accessible name is matched with the whitespace the runner normalizes away", () => { + const wrapped = [component({ bounds: TAB, role: "text", label: "This starter app includes example\ncode to help you get started." })]; + assert.deepEqual(resolveSelector(wrapped, { x: 50, y: 820 }).selector, { + kind: "text", text: "This starter app includes example code to help you get started.", + }); +}); + +/** + * The home screen of the Expo app under test, captured from the runner's own snapshot + * (`capture.snapshot({ interactiveOnly: false })` projected by `@e2e-dev/mobile`). + * + * Two things in it are the whole point of reading this tree. The tab bar's items arrive with role + * `tab` — the engine derives that from the tab bar around them, where iOS accessibility calls + * them radio buttons and a table here had to guess. And every piece of text is reported twice, + * parent and child, which the runner resolves innermost-first. + */ +const HOME_SCREEN = [ + component({ bounds: { x: 24, y: 300, width: 354, height: 30 }, role: "text", label: "GET STARTED", text: "GET STARTED", nodeId: "n4" }), + component({ bounds: { x: 24, y: 300, width: 354, height: 30 }, role: "text", label: "GET STARTED", text: "GET STARTED", nodeId: "n5", parentId: "n4" }), + component({ bounds: { x: 0, y: 790, width: 402, height: 84 }, role: "tablist", label: "Tab Bar", text: "Tab Bar", nodeId: "n18" }), + component({ bounds: { x: 20, y: 795, width: 94, height: 54 }, role: "tab", label: "Home", text: "Home", nodeId: "n19", parentId: "n18" }), + component({ bounds: { x: 154, y: 795, width: 94, height: 54 }, role: "tab", label: "Explore", text: "Explore", nodeId: "n20", parentId: "n18" }), + component({ bounds: { x: 288, y: 795, width: 94, height: 54 }, role: "tab", label: "Settings", text: "Settings", nodeId: "n21", parentId: "n18" }), +]; + +test("a tab bar item records as a tab, because that is the role the runner gave it", () => { + const { selector, ambiguous } = resolveSelector(HOME_SCREEN, { x: 200, y: 820 }, SCREEN); + assert.deepEqual(selector, { kind: "role", role: "tab", name: "Explore" }); + assert.equal(ambiguous, false); + assert.equal(selectorSource(selector), 'screen.getByRole("tab", "Explore")'); +}); + +test("text repeated by its own parent resolves without a TODO", () => { + const { selector, ambiguous } = resolveSelector(HOME_SCREEN, { x: 100, y: 310 }, SCREEN); + assert.deepEqual(selector, { kind: "text", text: "GET STARTED" }); + assert.equal(ambiguous, false); +}); + +/** + * The explore-list screen, captured the same way. This is the flow that failed: the heading, the + * navigation bar carrying the only testID on screen, and the search field with a placeholder and + * no name. + */ +const EXPLORE_LIST_SCREEN = [ + component({ bounds: { x: 0, y: 62, width: 402, height: 54 }, role: "navigation", identifier: "explore-list", nodeId: "n2" }), + component({ bounds: { x: 16, y: 62, width: 98, height: 44 }, role: "button", identifier: "BackButton", label: "(tabs)", text: "(tabs)", nodeId: "n3", parentId: "n2" }), + component({ bounds: { x: 24, y: 140, width: 354, height: 44 }, role: "text", label: "Explore Cities", text: "Explore Cities", nodeId: "n5" }), + component({ bounds: { x: 24, y: 200, width: 354, height: 46 }, role: "other", label: "NewYork City", text: "NewYork City", nodeId: "n6" }), + SEARCH_FIELD, +]; + +test("the recorded explore-list flow names every element the way the runner resolves it", () => { + const heading = resolveSelector(EXPLORE_LIST_SCREEN, { x: 100, y: 160 }, SCREEN); + assert.deepEqual(heading.selector, { kind: "text", text: "Explore Cities" }); + assert.equal(heading.ambiguous, false, "the heading must not carry a TODO"); + + const search = resolveSelector(EXPLORE_LIST_SCREEN, { x: 200, y: 490 }, SCREEN); + assert.deepEqual(search.selector, { kind: "placeholder", placeholder: "Search transactions" }); + + const back = resolveSelector(EXPLORE_LIST_SCREEN, { x: 50, y: 80 }, SCREEN); + assert.deepEqual(back.selector, { kind: "testId", testId: "BackButton" }); + + // An element the engine left as `other` has no role `getByRole` accepts, so it is named by what + // it says rather than by a role that would not compile. + const city = resolveSelector(EXPLORE_LIST_SCREEN, { x: 100, y: 220 }, SCREEN); + assert.deepEqual(city.selector, { kind: "text", text: "NewYork City" }); +}); + +test("the emitted explore-list test carries no link role and no TODO", () => { + const source = emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "role", role: "tab", name: "Explore" }, label: "" }, + { id: "2", kind: "tap", selector: { kind: "text", text: "Go To Explore List" }, label: "" }, + { id: "3", kind: "assertVisible", selector: { kind: "text", text: "Explore Cities" }, label: "" }, + { id: "4", kind: "tap", selector: { kind: "placeholder", placeholder: "Search transactions" }, label: "" }, + ], { title: "the explore list opens" }); + assert.match(source, /await screen\.getByPlaceholder\("Search transactions"\)\.tap\(\);/); + assert.match(source, /await expect\(screen\.getByText\("Explore Cities"\)\)\.toBeVisible\(\);/); + assert.doesNotMatch(source, /getByRole\("link"/); + assert.doesNotMatch(source, /TODO/); +}); + +test("an empty tree is reported rather than worked around", () => { + // The runner's snapshot is rooted at the application, so the simulator's own status bar is not + // in it and there is no "only the chrome" state left to detect. Nothing at all still is one. + assert.equal(treeHealth(EXPLORE_LIST_SCREEN), "ok"); + assert.equal(treeHealth([]), "no-accessibility"); +}); + +test("the emitter says which lines each step produced, comments and gates included", () => { + const { source, spans } = emitE2eTestLines([ + { id: "1", kind: "tap", selector: { kind: "role", role: "tab", name: "Explore" }, label: "" }, + { id: "2", kind: "tap", selector: { kind: "text", text: "Open" }, label: "", ambiguous: true }, + { id: "3", kind: "tap", selector: { kind: "point", point: { x: 40, y: 90 } }, label: "", anchor: { kind: "text", text: "Explore Cities" } }, + ], { title: "spans" }); + const lines = source.split("\n"); + // 1-based, as a file's own line numbers are, which is what the runner's report uses. + for (const span of spans) for (const line of span.calls) assert.match(lines[line - 1], /^ await /); + assert.deepEqual(spans.map(span => [span.from, span.to]), [[8, 8], [9, 10], [11, 13]]); + // An ambiguous step is a TODO plus a call; a coordinate tap is a TODO, a wait and a call. + assert.deepEqual(spans.map(span => span.calls), [[8], [10], [12, 13]]); + assert.equal(source, emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "role", role: "tab", name: "Explore" }, label: "" }, + { id: "2", kind: "tap", selector: { kind: "text", text: "Open" }, label: "", ambiguous: true }, + { id: "3", kind: "tap", selector: { kind: "point", point: { x: 40, y: 90 } }, label: "", anchor: { kind: "text", text: "Explore Cities" } }, + ], { title: "spans" }), "one emitter, so the preview and the spans can never describe different files"); +}); + +test("the emitter writes every new step kind as the runner's own API", () => { + const source = emitE2eTest([ + { id: "1", kind: "longPress", selector: { kind: "text", text: "Paris" }, label: "" }, + { id: "2", kind: "fill", selector: { kind: "placeholder", placeholder: "Search transactions" }, label: "", value: "Paris" }, + { id: "3", kind: "assertText", selector: { kind: "testId", testId: "total" }, label: "", value: "42" }, + { id: "4", kind: "scrollUntilVisible", selector: { kind: "text", text: "Images" }, label: "" }, + ], { title: "every kind" }); + assert.match(source, /await screen\.getByText\("Paris"\)\.longPress\(\);/); + assert.match(source, /await screen\.getByPlaceholder\("Search transactions"\)\.fill\("Paris"\);/); + assert.match(source, /await expect\(screen\.getByTestId\("total"\)\)\.toHaveText\("42"\);/); + assert.match(source, /await screen\.scrollUntilVisible\(screen\.getByText\("Images"\)\);/); +}); + +test("the element menu offers only what the runner can do with that element", () => { + const button = component({ bounds: TAB, role: "button", label: "Go To Explore List", text: "Go To Explore List" }); + const rows = stepOffers({ kind: "text", text: "Go To Explore List" }, button); + // One row per kind of thing to do. Enumerating `longPress` beside `tap`, and `toHaveText` beside + // `toBeVisible`, made the menu a list of near-duplicates of one selector. + assert.deepEqual(rows.map(row => row.source), [ + 'screen.getByText("Go To Explore List").tap()', + 'expect(screen.getByText("Go To Explore List")).toBeVisible()', + 'screen.scrollUntilVisible(screen.getByText("Go To Explore List"))', + ]); + // A button takes no typed text, so there is no `fill` row to get wrong. + assert.equal(rows.some(row => row.kind === "fill"), false); + assert.deepEqual(rows.filter(row => row.needsValue).map(row => row.kind), []); +}); + +test("a text field is offered typing, and a coordinate is offered nothing but the tap", () => { + const field = component({ bounds: TAB, role: "textbox", placeholder: "Search transactions", value: "Paris" }); + const rows = stepOffers({ kind: "placeholder", placeholder: "Search transactions" }, field); + const fill = rows.find(row => row.kind === "fill"); + assert.equal(fill?.source, 'screen.getByPlaceholder("Search transactions").fill("Paris")'); + assert.equal(fill?.needsValue, true, "the panel asks for the text rather than typing a guess"); + assert.equal(fill?.suggestedValue, "Paris"); + // A coordinate has no element to be visible, to hold text or to be scrolled to. + assert.deepEqual(stepOffers({ kind: "point", point: { x: 10, y: 20 } }).map(row => row.kind), ["tap"]); +}); + +test("the full command list is the rest of the runner's surface, gated on what the element carries", () => { + const button = component({ bounds: TAB, role: "button", label: "Go To Explore List", text: "Go To Explore List" }); + const rows = allStepOffers({ kind: "text", text: "Go To Explore List" }, button); + // The short list stays short; this is where a step the runner has but the four rows do not — an + // element that should be gone, a disabled button — is reachable without leaving the panel. + // The four swipes are one command offered four ways; the menu gathers them into a single row. + assert.deepEqual(rows.map(row => row.kind), [ + "tap", "doubleTap", "longPress", "swipe", "swipe", "swipe", "swipe", "scrollUntilVisible", + "assertVisible", "assertNotVisible", "assertText", "assertContainsText", + "assertEnabled", "assertDisabled", "assertSelected", + ]); + assert.equal(rows.find(row => row.kind === "assertNotVisible")?.source, + 'expect(screen.getByText("Go To Explore List")).not.toBeVisible()'); + assert.equal(rows.find(row => row.kind === "doubleTap")?.source, 'screen.getByText("Go To Explore List").doubleTap()'); + assert.equal(rows.find(row => row.kind === "assertText")?.suggestedValue, "Go To Explore List", + "a text comparison starts from the text the element actually has"); + // A button has no value to fill or to compare against, and the checked matchers are left out + // because whether a device reports a `checked` state at all depends on the platform. + assert.equal(rows.some(row => row.kind === "fill" || row.kind === "assertValue"), false); + assert.equal(rows.some(row => String(row.kind).includes("Checked")), false); + + const field = component({ bounds: TAB, role: "textbox", placeholder: "Search transactions", value: "Paris" }); + const fieldRows = allStepOffers({ kind: "placeholder", placeholder: "Search transactions" }, field); + assert.equal(fieldRows.find(row => row.kind === "assertValue")?.source, + 'expect(screen.getByPlaceholder("Search transactions")).toHaveValue("Paris")'); + assert.equal(fieldRows.find(row => row.kind === "assertValue")?.needsValue, true); + // A coordinate names no element, so nothing that is about one is offered for it — but it does + // have a viewport, and `screen.swipe` is about the viewport rather than about anything in it. + assert.deepEqual(allStepOffers({ kind: "point", point: { x: 10, y: 20 } }).map(row => row.kind), + ["tap", "swipe", "swipe", "swipe", "swipe"]); +}); + +test("new step kinds read back to the user with the value they carry", () => { + assert.equal(stepLabel("fill", { kind: "placeholder", placeholder: "Search" }, "Paris"), 'Type into "Search" → "Paris"'); + assert.equal(stepLabel("longPress", { kind: "text", text: "Paris" }), 'Long press "Paris"'); + assert.equal(stepLabel("scrollUntilVisible", { kind: "text", text: "Images" }), 'Scroll to "Images"'); +}); + +test("a unique candidate beats an ambiguous preferred one, for this element or an enclosing one", () => { + // The tab bar's Settings tab, as the runner reports it: a `tab` role with a name, inside a + // tablist, beside a heading that carries the same text. Captured from the live screen. + const heading = component({ bounds: { x: 140, y: 188, width: 121, height: 44 }, role: "text", label: "Settings", text: "Settings", nodeId: "n3" }); + const echoed = { ...heading, nodeId: "n4", parentId: "n3" }; + const tab = component({ bounds: { x: 240, y: 795, width: 94, height: 54 }, role: "tab", label: "Settings", text: "Settings", nodeId: "n13", parentId: "n10" }); + const components = [heading, echoed, tab]; + + // On the tab, `getByText("Settings")` matches two and `getByRole("tab", "Settings")` matches one. + // Role is the earlier rank *and* the unique one, so the ambiguity never reaches the file. + const onTab = resolveSelector(components, { x: 287, y: 820 }, SCREEN); + assert.deepEqual(onTab.selector, { kind: "role", role: "tab", name: "Settings" }); + assert.equal(onTab.ambiguous, false); + assert.equal(onTab.matches, 1); +}); + +test("an element the runner cannot name uniquely is flagged with the count, not a verdict", () => { + // The same screen's heading. Its role is `text`, which `getByRole` does not accept, and the only + // thing left — its text — is shared with the tab. The tree gives the two no distinguishing + // ancestor either, so there is no scoped selector to fall back to: a testID is the way out. + const heading = component({ bounds: { x: 140, y: 188, width: 121, height: 44 }, role: "text", label: "Settings", text: "Settings", nodeId: "n3", parentId: "n2" }); + const echoed = { ...heading, nodeId: "n4", parentId: "n3" }; + const tab = component({ bounds: { x: 240, y: 795, width: 94, height: 54 }, role: "tab", label: "Settings", text: "Settings", nodeId: "n13", parentId: "n10" }); + + const resolved = resolveSelector([heading, echoed, tab], { x: 200, y: 210 }, SCREEN); + + // `getByText("Settings")` matches the heading and the tab, in that order, and the heading is the + // one that was pointed at. `.nth(0)` runs on the device where the bare selector fails with + // LOCATOR_AMBIGUOUS, which is the difference between a recorded test and a recorded chore. + assert.deepEqual(resolved.selector, { kind: "nth", inner: { kind: "text", text: "Settings" }, index: 0, matches: 2 }); + assert.equal(resolved.ambiguous, true); + assert.equal(resolved.matches, 2, "the count is what a user can act on"); + assert.equal(selectorSource(resolved.selector), 'screen.getByText("Settings").nth(0)'); + + // The tab, the second match, records the second index — from the resolved node, not an assumption. + const onTab = resolveSelector([heading, echoed, tab], { x: 287, y: 820 }, SCREEN); + assert.deepEqual(onTab.selector, { kind: "role", role: "tab", name: "Settings" }, "the tab has a unique name of its own"); +}); + +test("a narrowed step emits a note that it runs, and names the durable fix", () => { + const source = emitE2eTest([ + { id: "1", kind: "assertVisible", label: "", ambiguous: true, + selector: { kind: "nth", inner: { kind: "text", text: "Settings" }, index: 0, matches: 2 } }, + ], { title: "narrowed" }); + assert.match(source, /await expect\(screen\.getByText\("Settings"\)\.nth\(0\)\)\.toBeVisible\(\);/); + // A note, not a TODO: nothing has to be edited for the test to work. + assert.match(source, /\/\/ 2 elements match this selector; this is the one that was recorded\. A testID would pin it by name\./); + assert.doesNotMatch(source, /TODO/); +}); + +test("a narrowed selector still reads back, and still supports every step", () => { + const selector = { kind: "nth" as const, inner: { kind: "text" as const, text: "Settings" }, index: 1, matches: 2 }; + assert.equal(stepLabel("tap", selector), 'Tap "Settings" (2 of 2)'); + // The narrowing is on the locator, so every action and matcher chains after it. + assert.deepEqual(stepOffers(selector, component({ bounds: TAB, role: "text", label: "Settings", text: "Settings" })).map(row => row.source), [ + 'screen.getByText("Settings").nth(1).tap()', + 'expect(screen.getByText("Settings").nth(1)).toBeVisible()', + 'screen.scrollUntilVisible(screen.getByText("Settings").nth(1))', + ]); +}); + +test("an agentic step emits the agent call and nothing about a locator", () => { + const source = emitE2eTest([ + { id: "1", kind: "agentAct", instruction: 'tap the "Explore" tab', label: "" }, + { id: "2", kind: "agentAssert", instruction: "the city list is showing", label: "" }, + ], { title: "the agent explores" }); + assert.match(source, /await agent\.act\("tap the \\"Explore\\" tab"\);/); + // `vision` defaults to false, so an assert that wants the default writes no options object. + assert.match(source, /await agent\.assert\("the city list is showing"\);/); + // There is no locator to be ambiguous, to narrow by position or to gate behind an anchor. + assert.doesNotMatch(source, /TODO/); + assert.doesNotMatch(source, /screen\./); +}); + +test("an agentic assertion emits the vision mode only when it is not the default", () => { + const emit = (vision?: VisionMode) => emitE2eTest( + [{ id: "1", kind: "agentAssert", instruction: "the badge is red", label: "", ...(vision === undefined ? {} : { vision }) }], + { title: "vision" }); + assert.match(emit(), /agent\.assert\("the badge is red"\);/); + assert.match(emit(false), /agent\.assert\("the badge is red"\);/); + assert.match(emit(true), /agent\.assert\("the badge is red", \{ vision: true \}\);/); + assert.match(emit("only"), /agent\.assert\("the badge is red", \{ vision: "only" \}\);/); +}); + +test("the agent fixture and the header comment follow the steps", () => { + const pinned: RecordedStep[] = [{ id: "1", kind: "tap", selector: { kind: "text", text: "Explore" }, label: "" }]; + const mixed: RecordedStep[] = [...pinned, { id: "2", kind: "agentAct", instruction: "scroll to the bottom", label: "" }]; + // A recording with no agentic step emits byte for byte what it always did. + const before = emitE2eTest(pinned, { title: "t" }); + assert.match(before, /test\("t", async \(\{ app, screen \}\) => \{/); + assert.match(before, /Every step is a pinned locator, driven by no agent\./); + const after = emitE2eTest(mixed, { title: "t" }); + assert.match(after, /test\("t", async \(\{ agent, app, screen \}\) => \{/); + assert.match(after, /Pinned locators where an element was named, and agent steps where it was described\./); + assert.doesNotMatch(after, /driven by no agent/); +}); + +test("an agent step reads as its instruction and the element only drafts one", () => { + assert.equal(agentStepLabel("agentAct", "tap the cart"), 'Agent "tap the cart"'); + assert.equal(agentStepLabel("agentAssert", "the cart is empty"), 'Agent check "the cart is empty"'); + // The suggestion is a typing shortcut written out of what the element carried; it is never the + // step, and an element with nothing to call it hands the user a blank field rather than a guess. + assert.equal(agentActSuggestion({ label: "Explore", role: "tab" }), 'tap the "Explore" tab'); + assert.equal(agentActSuggestion({ text: "Submit" }), 'tap "Submit"'); + // `other` is not a role `getByRole` accepts, so it is not a word to call the element by either. + assert.equal(agentActSuggestion({ label: "Submit", role: "other" }), 'tap "Submit"'); + assert.equal(agentActSuggestion({ role: "button" }), ""); + assert.equal(agentActSuggestion(), ""); +}); + +test("act params are emitted as literals, and as nothing at all when there are none", () => { + const act = (params?: Record<string, string>) => emitE2eTest( + [{ id: "1", kind: "agentAct", instruction: "sign in", label: "", ...(params ? { params } : {}) }], + { title: "params" }); + // The whole options object is omitted when there is nothing to put in it, so the line a + // recording without params emits is the line it emitted before params existed. + assert.match(act(), /await agent\.act\("sign in"\);/); + assert.match(act({}), /await agent\.act\("sign in"\);/); + // A row the user started and never named is not a param. + assert.match(act({ " ": "orphan" }), /await agent\.act\("sign in"\);/); + + // The text is the type: what parses as a number or a keyword is written as one. + assert.match(act({ email: "a@b.com", attempts: "3", retry: "false", note: "null" }), + /agent\.act\("sign in", \{ params: \{ email: "a@b\.com", attempts: 3, retry: false, note: null \} \}\);/); + assert.match(act({ n: "-2.5e3" }), /params: \{ n: -2\.5e3 \}/); + // Surrounding whitespace is not part of a number, and is kept where it is part of a string. + assert.match(act({ n: " 7 " }), /params: \{ n: 7 \}/); + assert.match(act({ s: " hi " }), /params: \{ s: " hi " \}/); + // Everything `Number()` would take and JSON would not stays a string, so the file says what the + // user typed rather than a different number or an identifier. + for (const value of ["0x10", "1_000", "Infinity", "NaN", "+1", "01", ""]) { + assert.match(act({ v: value }), new RegExp(`params: \\{ v: ${JSON.stringify(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\}`), + `${JSON.stringify(value)} is not a JSON number`); + } + // A key an object literal cannot take bare is quoted rather than mangled. + assert.match(act({ "user-id": "7" }), /params: \{ "user-id": 7 \}/); +}); + +test("only act carries params, and only assert carries vision", () => { + const source = emitE2eTest([ + { id: "1", kind: "agentAct", instruction: "sign in", label: "", params: { email: "a@b.com" } }, + { id: "2", kind: "agentAssert", instruction: "the dashboard is showing", label: "", vision: true }, + ], { title: "both" }); + assert.match(source, /agent\.act\("sign in", \{ params: \{ email: "a@b\.com" \} \}\);/); + assert.match(source, /agent\.assert\("the dashboard is showing", \{ vision: true \}\);/); + assert.doesNotMatch(source, /act\([^)]*vision/); + assert.doesNotMatch(source, /assert\([^)]*params/); +}); + +test("waitFor judges like assert and takes the same vision", () => { + const emit = (vision?: VisionMode) => emitE2eTest( + [{ id: "1", kind: "agentWaitFor", instruction: "the spinner is gone", label: "", ...(vision === undefined ? {} : { vision }) }], + { title: "wait" }); + assert.match(emit(), /await agent\.waitFor\("the spinner is gone"\);/); + assert.match(emit(false), /await agent\.waitFor\("the spinner is gone"\);/); + assert.match(emit(true), /await agent\.waitFor\("the spinner is gone", \{ vision: true \}\);/); + assert.match(emit("only"), /await agent\.waitFor\("the spinner is gone", \{ vision: "only" \}\);/); + // `interval` and `maxModelCalls` are budgets the panel never writes, so neither reaches the file. + assert.doesNotMatch(emit(true), /interval|maxModelCalls|timeout/); + // It is an agent step like the others, so it flips the fixtures and the header comment. + assert.match(emit(), /test\("wait", async \(\{ agent, app, screen \}\) => \{/); + assert.match(emit(), /Pinned locators where an element was named/); + assert.doesNotMatch(emit(), /TODO|screen\.getBy/); +}); + +test("a swipe is scoped to the element it names, or to the viewport when nothing is named", () => { + const swipe = (selector: RecordedSelector, direction?: "up" | "down" | "left" | "right") => emitE2eTest( + [{ id: "1", kind: "swipe", selector, label: "", ...(direction ? { direction } : {}) }], { title: "s" }); + for (const direction of ["up", "down", "left", "right"] as const) { + assert.match(swipe({ kind: "testId", testId: "carousel" }, direction), + new RegExp(`await screen\\.getByTestId\\("carousel"\\)\\.swipe\\(\\{ direction: "${direction}" \\}\\);`)); + assert.match(swipe({ kind: "point", point: { x: 4, y: 8 } }, direction), + new RegExp(`await screen\\.swipe\\(\\{ direction: "${direction}" \\}\\);`)); + } + // `momentum` defaults to `none` and how hard a flick was thrown is not something the panel can + // observe, so `direction` is the whole of the options object. + assert.doesNotMatch(swipe({ kind: "text", text: "Cards" }, "left"), /momentum/); + // A viewport swipe needs no element, so it is emitted rather than skipped. It keeps the wait + // every point step carries — it is as actionable as a tap and would fire into a splash screen — + // but not the TODO, which is about a coordinate breaking on a layout change and names no + // coordinate here. + const viewport = emitE2eTest([{ + id: "1", kind: "swipe", selector: { kind: "point", point: { x: 4, y: 8 } }, label: "", + direction: "down", anchor: { kind: "text", text: "Explore" }, + }], { title: "s" }); + assert.doesNotMatch(viewport, /TODO/); + assert.match(viewport, /await expect\(screen\.getByText\("Explore"\)\)\.toBeVisible\(\);\n\s+await screen\.swipe\(\{ direction: "down" \}\);/); + assert.doesNotMatch(viewport, /Skipped a step on a point/); +}); + +test("the swipe rows are one command offered four ways, on an element and on a bare point", () => { + const named = allStepOffers({ kind: "testId", testId: "carousel" }).filter(offer => offer.kind === "swipe"); + assert.deepEqual(named.map(offer => offer.suggestedValue), ["up", "down", "left", "right"]); + assert.equal(named[0].source, 'screen.getByTestId("carousel").swipe({ direction: "up" })'); + // Choosing a direction records the step at once; it is a choice from four, not text to supply. + assert.ok(named.every(offer => offer.needsValue === undefined)); + const point = allStepOffers({ kind: "point", point: { x: 4, y: 8 } }); + assert.deepEqual(point.map(offer => offer.kind), ["tap", "swipe", "swipe", "swipe", "swipe"]); + assert.equal(point[1].source, 'screen.swipe({ direction: "up" })'); +}); + +test("a swipe step says which way it went", () => { + assert.equal(stepLabel("swipe", { kind: "role", role: "list", name: "Cards" }, "left"), 'Swipe left on "Cards" (list)'); + assert.equal(stepLabel("swipe", { kind: "point", point: { x: 4, y: 8 } }, "down"), "Swipe down (viewport)"); + assert.equal(stepLabel("swipe", { kind: "testId", testId: "c" }), "Swipe up on #c", "the direction the menu lists first"); +}); + +test("an element names itself the same way a tap on it would", () => { + // The two entry points are one ranking. A point reaches the chain by hit-testing, an agent + // reaches it by walking `parentId` up, and both then run `resolveChain` — which is what keeps an + // agent-recorded step and a hand-recorded step the same step. + const components = [ + component({ bounds: TAB, role: "tablist", label: "Tab Bar", nodeId: "bar" }), + component({ bounds: { x: 0, y: 800, width: 50, height: 40 }, role: "tab", label: "Explore", nodeId: "tab", parentId: "bar" }), + ]; + const byPoint = resolveSelector(components, { x: 20, y: 820 }, SCREEN); + const byElement = resolveComponentSelector(components, components[1]); + assert.deepEqual(byPoint.selector, { kind: "role", role: "tab", name: "Explore" }); + assert.deepEqual(byElement?.selector, byPoint.selector); + assert.equal(byElement?.component, components[1]); +}); + +test("an element whose own node carries nothing is named by what encloses it", () => { + // The same depth-first rule the ladder follows: the innermost element that can be named at all + // wins, and a bare container falls through to its parent rather than to nothing. + const components = [ + component({ bounds: TAB, identifier: "SubmitButton", nodeId: "button" }), + component({ bounds: TAB, role: "other", nodeId: "icon", parentId: "button" }), + ]; + assert.deepEqual(resolveComponentSelector(components, components[1])?.selector, { kind: "testId", testId: "SubmitButton" }); +}); + +test("an element nothing can name resolves to nothing rather than to a coordinate", () => { + // The one place the two entry points differ, and deliberately: a tap has a coordinate to fall + // back on, while a `ref` does not — recording `screen.tapAt` from an element an agent named + // would be a worse step than the `agent.act` sentence it writes instead. + const components = [component({ bounds: TAB, role: "other", nodeId: "blank" })]; + assert.equal(resolveComponentSelector(components, components[0]), undefined); + assert.deepEqual(resolveSelector(components, { x: 50, y: 820 }).selector, { kind: "point", point: { x: 50, y: 820 } }); +}); + +test("the ancestor chain is innermost first and survives a broken parent link", () => { + const components = [ + component({ bounds: TAB, label: "Card", nodeId: "card" }), + component({ bounds: TAB, label: "Row", nodeId: "row", parentId: "card" }), + component({ bounds: TAB, label: "Orphan", nodeId: "orphan", parentId: "gone" }), + ]; + assert.deepEqual(ancestorChain(components, components[1]).map(item => item.nodeId), ["row", "card"]); + assert.deepEqual(ancestorChain(components, components[2]).map(item => item.nodeId), ["orphan"]); +}); + +test("a swipe is drawn well inside the box, so the OS does not read it as an edge gesture", () => { + // Both drivers take a recorded swipe — the panel through the mirror's stream, the server through + // agent-device — so the arithmetic is shared rather than written twice. + const path = swipePath({ x: 0, y: 0, width: 400, height: 800 }, "up"); + assert.deepEqual(path, { from: { x: 200, y: 680 }, to: { x: 200, y: 120 } }); + assert.deepEqual(swipePath({ x: 0, y: 0, width: 400, height: 800 }, "right"), + { from: { x: 60, y: 400 }, to: { x: 340, y: 400 } }); + assert.equal(swipeDirection("sideways"), "up", "an unknown direction is the one the menu lists first"); + assert.equal(swipeDirection("left"), "left"); +}); + +test("the screen's own root is never what names an element", () => { + // The point path never reaches it, because the hit ladder drops anything spanning the screen. + // Without the same rule on the element path, every unnameable element on an app whose root + // carries a title resolved to that title — `screen.getByText("expoapp57")` for a scroll bar, + // reported as unique. + const components = [ + component({ bounds: { x: 0, y: 0, ...SCREEN }, label: "expoapp57", role: "application", nodeId: "root" }), + component({ bounds: TAB, role: "other", nodeId: "scrollbar", parentId: "root", label: "" }), + ]; + assert.equal(resolveComponentSelector(components, components[1], SCREEN), undefined); + assert.equal(resolveComponentSelector(components, components[0], SCREEN), undefined); + // Without a screen size there is nothing to compare against, so the rule cannot apply. + assert.deepEqual(resolveComponentSelector(components, components[1])?.selector, { kind: "text", text: "expoapp57" }); +}); + +test("a viewport swipe carries no coordinate, so it carries no coordinate TODO", () => { + // `screen.swipe({ direction })` names no point: there is nothing to break on a layout change + // and nothing to wait for. The TODO and the anchor gate are about a coordinate *tap*. + const step: RecordedStep = { + id: "s", kind: "swipe", label: "Swipe up (viewport)", direction: "up", + selector: { kind: "point", point: { x: 0, y: 0 } }, + }; + const { source } = emitE2eTestLines([step], { title: "t" }); + assert.match(source, /await screen\.swipe\(\{ direction: "up" \}\);/); + assert.doesNotMatch(source, /TODO/); + // A coordinate *tap* still says what it is. + const tap: RecordedStep = { id: "s", kind: "tap", label: "Tap at 0, 0", selector: { kind: "point", point: { x: 0, y: 0 } } }; + assert.match(emitE2eTest([tap], { title: "t" }), /TODO: nothing resolved here/); +}); + +test("the screen's own root is never the thing a coordinate step waits for", () => { + // It is the nearest element to any point and it names nothing: + // `expect(screen.getByText("<the app>")).toBeVisible()` waits for the application to exist, + // which it did before the test started. + const components = [ + component({ bounds: { x: 0, y: 0, ...SCREEN }, label: "expoapp57", role: "application", nodeId: "root" }), + component({ bounds: { x: 100, y: 400, width: 80, height: 40 }, label: "Paris", role: "text", nodeId: "paris" }), + ]; + assert.deepEqual(anchorSelector(components, { x: 201, y: 437 }, SCREEN), { kind: "text", text: "Paris" }); +}); + +/** + * The Android mirror and the runner's tree do not measure in the same unit: agent-device reports + * element bounds in device pixels, and scrcpy caps the stream's longest edge, so a 1280x2856 + * emulator arrives as 574x1280. Left unconverted, a pointer anywhere below the top fifth of the + * mirror falls outside every element but the full-screen groups — which is the one element the + * recorder offered, and the outline it drew was the right rectangle at twice the size. + */ +test("an Android tree scaled into the mirror's space resolves the element under the pointer", () => { + const device = { width: 1280, height: 2856 }; + const mirror = { width: 574, height: 1280 }; + // Captured from a booted AVD through the recorder's own read path. + const tree: ScreenComponent[] = [ + { name: "root", role: "group", bounds: { x: 0, y: 0, width: 1280, height: 2856 }, depth: 0, source: "accessibility", nodeId: "n1" }, + { name: "Explore", role: "tab", label: "Explore", text: "Explore", bounds: { x: 427, y: 2544, width: 427, height: 240 }, depth: 1, source: "accessibility", nodeId: "n2", parentId: "n1" }, + ]; + const point = { x: 287, y: 1194 }; + + assert.equal(resolveSelector(tree, point, mirror).component, undefined, "unconverted, the tab is nowhere near the pointer"); + + const scaled = scaledComponents(tree, device, mirror); + const tab = scaled[1]!; + assert.equal(Math.round(tab.bounds.x), 191); + assert.equal(Math.round(tab.bounds.y), 1140); + assert.equal(Math.round(tab.bounds.width), 191); + assert.equal(Math.round(tab.bounds.height), 108); + assert.equal(resolveSelector(scaled, point, mirror).component?.nodeId, "n2"); +}); + +test("components are left alone when the mirror already measures what the runner measured", () => { + const screen = { width: 402, height: 874 }; + const tree: ScreenComponent[] = [ + { name: "Explore", role: "tab", bounds: { x: 100, y: 800, width: 100, height: 40 }, depth: 0, source: "accessibility", nodeId: "n1" }, + ]; + assert.deepEqual(scaledComponents(tree, screen, screen), tree); + // A screen the runner could not name must not silently collapse every rectangle to zero. + assert.deepEqual(scaledComponents(tree, { width: 0, height: 0 }, screen), tree); +}); diff --git a/packages/skills/README.md b/packages/skills/README.md index 3d5af9bc..9b99593d 100644 --- a/packages/skills/README.md +++ b/packages/skills/README.md @@ -3,10 +3,13 @@ Agent skills for the tools every host's server exposes. Each skill is a standard `SKILL.md` with its references; `agents/openai.yaml` adds Codex's display metadata, which other hosts' packages leave out. A host's package step includes the skills its host supports: -the Codex and Cursor plugins ship `mobile-dev`, `mobile-dev-setup` and `agent-device`. +the Codex and Cursor plugins ship `mobile-dev`, `mobile-dev-setup`, `mobile-dev-testing` and +`agent-device`. The Claude Code plugin ships its own `mobile-dev` skill from `apps/claude/skills`, for -its server without the panel, and the shared `agent-device` skill; Claude Desktop's -MCPB bundles have no skill support yet. +its server without the panel, and the shared `agent-device` and `mobile-dev-testing` +skills; Claude Desktop's MCPB bundles have no skill support yet. `mobile-dev-testing` +is shared rather than per-host: recording a test is the same five tools and the same +panel wherever the server runs. `agent-device/references/workflow.md` is the pinned agent-device runtime's own `help workflow`. `npm run catalog:agent-device` rewrites it with the tool catalog, and a diff --git a/packages/skills/mobile-dev-testing/SKILL.md b/packages/skills/mobile-dev-testing/SKILL.md new file mode 100644 index 00000000..0ba95546 --- /dev/null +++ b/packages/skills/mobile-dev-testing/SKILL.md @@ -0,0 +1,124 @@ +--- +name: mobile-dev-testing +description: Use when the user asks to record, write or capture an end-to-end test for a screen or a flow of their iOS or Android app, with Mobile Dev's Testing panel. Drives the simulator or emulator through Mobile Dev's recorder one step at a time, pinning each step to a locator the e2e runner can resolve, and leaves the flow in the panel for the user to adjust, save and run. Skip when the user wants a test written straight into a file, or when there is no Mobile Dev panel. +--- + +# Record an e2e test with Mobile Dev + +The user asks for a test for a screen — *"record the test for the important areas on the Explore +Details screen"* — and you write it by driving their simulator or emulator, one step per tool call. The steps +appear in Mobile Dev's **Testing** tab as you record them, and when you stop, the flow is sitting +there unsaved for the user to adjust. + +You record. The user saves and runs. Those two are theirs, and the tools to do them are not yours +to call. + +## Before you start + +The recording goes against whatever the Testing panel already has selected: an **e2e project**, a +**booted iOS simulator or Android emulator in the mirror**, and a **target with a bundle id** in that project's +`e2e.config.ts`. `mobile_testing_agent_begin` reads that selection and refuses, naming what is +missing, when any of the three is unset. + +If it refuses, say which one is missing and ask the user to set it in Mobile Dev's Testing tab. +**Never choose the project for them** — the picker is a system folder dialog, which must not open +in the middle of your turn — and do not reach for another device because the chosen one is not +booted. Open the panel with `mobile_open_simulator` if it is not open, and otherwise stop and ask. + +The app should already be on screen. The recorder attaches to the running app and never relaunches +it, so a flow recorded from a cold home screen is a flow the user will have to fix. + +## The loop + +1. `mobile_testing_agent_begin({ title, goal })` — one short title for the test and one sentence + on what the flow covers. The goal is what the user sees in the panel's banner, so write it for + them. +2. `mobile_testing_agent_screen({})` — the element tree of the screen you are on. One row per + element with its role, name, text, placeholder, value, testId, state and bounds, **and the + selector a step on it would emit**, its rank, and whether it is unique. +3. Pick a `ref` from that read and call `mobile_testing_agent_step`. +4. **Read the screen again after every step that acts.** Every tap, fill and swipe changes the + screen, so the refs from the last read go with it; a stale `ref` is refused rather than acted + on. An assertion changes nothing, so several in a row need only one read. +5. `mobile_testing_agent_finish({ note })` when the flow is complete, and tell the user the flow + is waiting in the Testing panel. + +Use `mobile_testing_agent_abort({ reason })` if you cannot finish — a screen you cannot reach, an +app that crashed. Whatever you recorded stays in the panel for the user. + +## Never write a locator + +You name a `ref`; the panel resolves the selector. The read already tells you what it would emit +and whether it is unique, which is the only thing you need to judge: + +| The read says | What to do | +| --- | --- | +| `unique: true` | Record the step on that `ref`. | +| `unique: false` | The selector matches more than one element and the step is pinned by position. Prefer a different element that is unique — the row's parent, a nearby testId — and only then record this one. | +| no `selector` at all | Nothing on the screen names that element. Record it as a sentence: `mobile_testing_agent_step` with an `instruction` and no `ref`. | +| `offScreen: true` | It is below the fold or behind the keyboard. `getByText` still resolves it, so it is fine to **assert** on; a tap on it is refused by the device. Record a swipe first if you need to act on it. | + +A sentence becomes `agent.act` or `agent.assert`, which the runner resolves with a model at run +time. It is the fallback, not the tool: the point of a recorded test is that it is deterministic, +and a flow that is half sentences is a flow the user could have written by prompting. If you reach +for more than one or two in a flow, the screen needs testIDs and that is worth saying. + +## What to record + +The test starts from `app.open()`, so **the navigation to the screen is part of the flow** and is +recorded like everything else: tab, button, row, until you are where the user asked for. + +Then, on the target screen: + +- its **landmarks** as assertions — the heading, the section titles, the one piece of copy that + says the right data loaded; +- its **one or two real affordances** exercised — the search field filled, the filter tapped, the + disclosure opened; +- the **result** of each asserted, which is what makes it a test rather than a transcript. + +Keep a flow under about fifteen steps. A long flow is two tests, and the second one is a second +`begin` after you have finished the first. + +A control you **open** is part of the screen, so assert what it revealed — and close it again if +the screen is meant to be left as you found it. A disclosure left open is a screen the next step +resolves against and the test will not reach. + +## Where the flow ends + +**The screen the user named is the scope of the recording, not the place it starts.** Getting there +is recorded because the test has to start from `app.open()`; everything after you leave it is +somebody else's test. + +- You may tap a control that **navigates away**, and assert the arrival with **one** assertion — + that is how the screen's own navigation gets covered. Then the flow **ends**. Record nothing on + the screen you arrived at: not its heading, not its fields, not its search. +- A step's result says `navigated: true` when the action replaced the screen. Treat it as the + boundary it is: assert you arrived, then `mobile_testing_agent_finish`. +- **A flow name is not a boundary.** "Record the checkout flow", "record signing in" — the flow is + the scope, and crossing screens is the whole of it. A *screen* name is a boundary; a flow name is + not. When the user's words name neither clearly, ask rather than guess wide. +- Say what you left out instead of recording it: *"I stopped at Explore List — say the word and + I'll record that screen too."* A sentence costs nothing; six steps about the wrong screen cost + the user an edit. + +Assert text the screen actually carries, copied from the read — not text you expect it to carry. If +the thing you want to assert is not in the read at all, record `{ kind: "swipe", direction: "up" }` +with no `ref` to scroll the viewport and read again; if it still is not there, say so rather than +asserting text you have not seen. + +## What you never do + +- **Never save.** There is no tool for it in your hands, and finishing is the handover: the steps + are dirty in the panel and the user presses Save when they are happy with them. +- **Never run the test.** There is no file to run, and a run resets the app and takes the device + session the recording is being read through. +- **Never drive the device another way while recording** — not `device_*`, not `mobile_open_url`, + not the mirror. The recorder's steps and another driver's taps interleave into a flow that does + not describe anything. Everything the flow does goes through `mobile_testing_agent_step`, which + is what makes each action a recorded step. + +## When you are done + +Tell the user what you recorded, in their terms, and that it is **in the Testing panel, unsaved**. +Say which parts fell back to a sentence, if any, and why. Do not paste the emitted file as the +deliverable — the panel is showing it, live, next to the steps. diff --git a/packages/skills/mobile-dev-testing/agents/openai.yaml b/packages/skills/mobile-dev-testing/agents/openai.yaml new file mode 100644 index 00000000..40311bfc --- /dev/null +++ b/packages/skills/mobile-dev-testing/agents/openai.yaml @@ -0,0 +1,10 @@ +interface: + display_name: "Record an e2e test" + short_description: "Record an e2e test from the simulator" + default_prompt: "Use Mobile Dev and record the test for the important areas on this screen." + +dependencies: + tools: + - type: "mcp" + value: "mobile-dev" + description: "Bundled Baguette simulator tools and native panel" diff --git a/packages/ui/src/code-highlight.ts b/packages/ui/src/code-highlight.ts new file mode 100644 index 00000000..c3477086 --- /dev/null +++ b/packages/ui/src/code-highlight.ts @@ -0,0 +1,63 @@ +/** + * A tokenizer for the TypeScript the recorder emits. + * + * This is deliberately not a parser and not a general highlighter: the preview only ever shows a + * file this panel wrote, whose shape is fixed by `emitE2eTest`, so literals, line comments, + * keywords and call names are the whole vocabulary. A real highlighter would add a grammar and + * hundreds of kilobytes to a panel that renders one twenty-line file. + */ +export type CodeTokenKind = "comment" | "string" | "keyword" | "call" | "number" | "plain"; +export type CodeToken = { kind: CodeTokenKind; text: string }; + +const KEYWORDS = new Set([ + "import", "from", "export", "default", "const", "let", "var", "async", "await", "function", + "return", "new", "if", "else", "for", "of", "in", "true", "false", "null", "undefined", +]); + +// Line comments, then any quoted string, then a word, then a number. Anything else is plain text, +// which keeps whitespace and newlines intact for the <pre> that renders these. +const TOKEN = /(\/\/[^\n]*)|("(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'|`(?:[^`\\]|\\.)*`)|([A-Za-z_$][\w$]*)|(\d+(?:\.\d+)?)/g; + +export function highlightCode(source: string): CodeToken[] { + const tokens: CodeToken[] = []; + const push = (kind: CodeTokenKind, text: string) => { if (text) tokens.push({ kind, text }); }; + let last = 0; + for (const match of source.matchAll(TOKEN)) { + push("plain", source.slice(last, match.index)); + last = match.index + match[0].length; + const [, comment, string, word, number] = match; + if (comment !== undefined) push("comment", comment); + else if (string !== undefined) push("string", string); + else if (word !== undefined) push(KEYWORDS.has(word) ? "keyword" : source[last] === "(" ? "call" : "plain", word); + else push("number", number!); + } + push("plain", source.slice(last)); + return tokens; +} + +/** The class each kind renders with. Both themes, since the panel follows the host's. */ +export const CODE_TOKEN_CLASS: Record<CodeTokenKind, string> = { + comment: "text-muted-foreground italic", + string: "text-emerald-600 dark:text-emerald-400", + keyword: "text-violet-600 dark:text-violet-400", + call: "text-blue-600 dark:text-blue-400", + number: "text-amber-600 dark:text-amber-400", + plain: "", +}; + +/** + * The lines of `next` that `previous` did not have, as a `[start, end)` range. + * + * The preview is re-emitted whole on every change, so there is no "the step I just added" to + * point at — it has to be recovered by comparing. Matching from both ends finds it without a real + * diff: the emitter appends steps in order, so what differs is one contiguous run, and removing a + * step or renaming the test narrows to the same kind of range. + */ +export function changedLines(previous: readonly string[], next: readonly string[]): [number, number] { + let start = 0; + while (start < previous.length && start < next.length && previous[start] === next[start]) start++; + let end = 0; + while (end < previous.length - start && end < next.length - start + && previous[previous.length - 1 - end] === next[next.length - 1 - end]) end++; + return [start, next.length - end]; +} diff --git a/packages/ui/src/components/cascading-menu.tsx b/packages/ui/src/components/cascading-menu.tsx new file mode 100644 index 00000000..b13edc23 --- /dev/null +++ b/packages/ui/src/components/cascading-menu.tsx @@ -0,0 +1,235 @@ +import { createContext, useCallback, useContext, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { MARGIN, placeBelow, placeBeside } from "../popover-placement"; +import type { Box, Placement } from "../popover-placement"; + +/** + * The cascading half of the Testing panel's menus: a branch opens *beside* the row it hangs off, + * with the menu it came from still on screen. + * + * A branch that replaced what it was opened from read as navigation rather than as a menu — the + * user lost the element they were acting on, and the only way back was a button put there for the + * purpose. Beside is what a menu does, and it is also what makes a branch several levels deep + * legible: the trail from the element to the command is all on screen at once. + * + * Everything that makes that work rather than merely look right lives here: hover intent, so a row + * brushed past on the way somewhere else does not open; a safe path, so the pointer may travel + * diagonally to the panel without crossing a row that would close it; and the keyboard, so the + * branch is not reachable by pointer alone. + */ + +/** The window every popover here is fitted into. See `popover-placement`. */ +export const windowView = () => ({ width: window.innerWidth, height: window.innerHeight }); + +/** A portalled child branch keeps its parent open while the pointer is inside it. */ +const BranchHoverContext = createContext<{ hold: () => void; leave: () => void } | null>(null); + +export { MARGIN, placeBelow, placeBeside }; + +/** Re-fits a popover whenever its own size, its anchor or the window changes. */ +export function usePlacement( + element: React.RefObject<HTMLElement | null>, + place: () => Placement | undefined, + deps: unknown[], +): Placement | undefined { + const [placement, setPlacement] = useState<Placement>(); + const measure = useCallback(() => { + const next = place(); + // The same numbers mean the same position, and writing them again would re-run this effect. + setPlacement(current => current && next && current.left === next.left && current.top === next.top + && current.maxHeight === next.maxHeight ? current : next); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps); + useLayoutEffect(() => { + measure(); + const node = element.current; + // A row that opens a form, a list that loads: the popover's own height is what decides whether + // it still fits, so it is watched rather than measured once. + const observer = typeof ResizeObserver === "function" && node ? new ResizeObserver(() => measure()) : undefined; + if (node) observer?.observe(node); + window.addEventListener("resize", measure); + window.addEventListener("scroll", measure, true); + return () => { + observer?.disconnect(); + window.removeEventListener("resize", measure); + window.removeEventListener("scroll", measure, true); + }; + }, [element, measure]); + return placement; +} + +/** How long a row must be pointed at before its branch opens. */ +const OPEN_MS = 90; +/** How long the branch survives the pointer leaving it, when the pointer is not heading for it. */ +const CLOSE_MS = 140; +/** The same, when the pointer is travelling towards the open panel. See `aimsAt`. */ +const SAFE_MS = 500; +/** How far past the panel's own edges the safe path still counts, for a hurried diagonal. */ +const SAFE_SLACK = 28; + +/** + * Whether the pointer's last movement is heading into the open panel. + * + * This is the "safe path" a cascading menu needs. The panel sits to one side of its row, so the + * pointer travelling to it crosses whatever rows lie between — and each of those is a row that + * would otherwise take the branch away mid-journey. Rather than tracking a triangle, the movement + * is extended to the panel's near edge: if it arrives within the panel's own span, the pointer is + * on its way there and the branch is held open long enough to get there. + */ +function aimsAt(from: { x: number; y: number }, to: { x: number; y: number }, panel: DOMRect | undefined): boolean { + if (!panel) return false; + const dx = to.x - from.x; + if (dx === 0) return false; + // The near edge is whichever side the panel actually opened on. + const edge = to.x <= panel.left ? panel.left : to.x >= panel.right ? panel.right : undefined; + if (edge === undefined) return true; + const steps = (edge - to.x) / dx; + if (steps < 0) return false; + const y = to.y + (to.y - from.y) * steps; + return y >= panel.top - SAFE_SLACK && y <= panel.bottom + SAFE_SLACK; +} + +/** + * The open/close logic one branch shares between its row and its panel. + * + * Returned as handlers rather than wired to the DOM here, because the row belongs to the menu that + * owns it and the panel is a portal somewhere else entirely; the two are one hover target with a + * gap and a stack of other rows between them. + */ +export function useBranch(panel: React.RefObject<HTMLElement | null>) { + const [open, setOpen] = useState(false); + const timer = useRef<ReturnType<typeof setTimeout>>(undefined); + const trail = useRef<{ from: { x: number; y: number }; to: { x: number; y: number } }>(undefined); + const stop = useCallback(() => { if (timer.current) { clearTimeout(timer.current); timer.current = undefined; } }, []); + useEffect(() => stop, [stop]); + + // The pointer is followed only while the branch is open, which is the only time its direction + // decides anything. + useEffect(() => { + if (!open) { trail.current = undefined; return; } + const move = (event: PointerEvent) => { + const to = { x: event.clientX, y: event.clientY }; + trail.current = { from: trail.current?.to ?? to, to }; + }; + window.addEventListener("pointermove", move); + return () => window.removeEventListener("pointermove", move); + }, [open]); + + // `stop` first, so entering either half of an open branch cancels a close already in flight. + const enter = useCallback(() => { + stop(); + timer.current = setTimeout(() => setOpen(true), OPEN_MS); + }, [stop]); + const hold = useCallback(() => { stop(); setOpen(true); }, [stop]); + const leave = useCallback(() => { + stop(); + const heading = trail.current && aimsAt(trail.current.from, trail.current.to, panel.current?.getBoundingClientRect()); + timer.current = setTimeout(() => setOpen(false), heading ? SAFE_MS : CLOSE_MS); + }, [panel, stop]); + const close = useCallback(() => { stop(); setOpen(false); }, [stop]); + return { open, enter, hold, leave, close, openNow: hold }; +} + +/** The first thing inside a panel that the keyboard can land on. */ +function focusFirst(node: HTMLElement | null) { + node?.querySelector<HTMLElement>("input, button, [tabindex]:not([tabindex='-1'])")?.focus(); +} + +/** + * One branch's panel: a portal beside the row it hangs off. + * + * A portal rather than a child of the menu for two reasons that have not changed: a menu taller + * than the window scrolls, and a scrolling box clips what hangs out of it; and a branch has to be + * free to lie outside the menu, the stage and the panel alike. + */ +export function BranchPanel({ panelRef, row, width, label, align, flip, side, children, attributes, onHold, onLeave, onClose }: { + /** The panel's own node, which the branch measures for its safe path. */ + panelRef: React.RefObject<HTMLDivElement | null>; + row: React.RefObject<HTMLElement | null>; + width: number; + label: string; + align?: "top" | "bottom"; + flip?: boolean; + /** Which side the branch opens on by design; see `placeBeside`. */ + side?: "left" | "right"; + children: React.ReactNode; + attributes?: Record<string, string>; + onHold: () => void; + onLeave: () => void; + /** Dismisses the branch and hands the keyboard back to the row that opened it. */ + onClose: () => void; +}) { + const panel = panelRef; + const parent = useContext(BranchHoverContext); + const placement = usePlacement(panel, () => { + const anchor = row.current?.getBoundingClientRect(); + return anchor && placeBeside(anchor, { width, height: panel.current?.offsetHeight ?? 0 }, windowView(), + { ...(flip === undefined ? {} : { flip }), ...(align === undefined ? {} : { align }), ...(side === undefined ? {} : { side }) }); + }, [row, width, align, flip, side, children]); + // Opened from the keyboard, the branch is useless until the keyboard is in it. Opened by the + // pointer this is harmless: the element focused is the one the pointer is already over. + useEffect(() => { if (document.activeElement === row.current) focusFirst(panel.current); }, [row]); + return createPortal(<BranchHoverContext.Provider value={{ hold: onHold, leave: onLeave }}><div + ref={panel} + role="menu" aria-label={label} {...attributes} + className="pointer-events-auto fixed z-50 flex flex-col gap-1.5 overflow-y-auto rounded-xl border bg-popover p-1.5 text-popover-foreground shadow-lg [&_button:not(:disabled)]:cursor-pointer" + style={{ left: placement?.left ?? 0, top: placement?.top ?? 0, width, maxHeight: placement?.maxHeight, visibility: placement ? "visible" : "hidden" }} + onMouseEnter={() => { onHold(); parent?.hold(); }} + onMouseLeave={() => { onLeave(); parent?.leave(); }} + onPointerDown={event => event.stopPropagation()} + onContextMenu={event => event.preventDefault()} + // Left is back out of a branch, as it is in every cascading menu, and Escape is the same door. + onKeyDown={event => { + if (event.key !== "Escape" && event.key !== "ArrowLeft") return; + // A text field owns its own arrows; only Escape leaves from inside one. + if (event.key === "ArrowLeft" && (event.target as HTMLElement).tagName === "INPUT") return; + event.stopPropagation(); + onClose(); + row.current?.focus(); + }} + >{children}</div></BranchHoverContext.Provider>, document.body); +} + +/** + * A row and the branch it opens, as one thing. + * + * The row renders through `renderRow` so each menu keeps its own row component, its tooltips and + * its test attributes; everything about *when* the branch is open is here, so it behaves the same + * in the mirror's menu as in any other. + */ +export function Branch({ label, width, align, flip, side, renderRow, children, attributes }: { + label: string; + width: number; + align?: "top" | "bottom"; + flip?: boolean; + /** Which side the branch opens on by design; see `placeBeside`. */ + side?: "left" | "right"; + renderRow: (handlers: { onOpen: () => void; onClick: () => void }) => React.ReactNode; + children: React.ReactNode; + attributes?: Record<string, string>; +}) { + const panel = useRef<HTMLDivElement>(null); + const row = useRef<HTMLDivElement>(null); + const { open, enter, hold, leave, close, openNow } = useBranch(panel); + return <> + <div + ref={row} + className="flex flex-col" + onMouseEnter={enter} + onMouseLeave={leave} + // A branch opens on hover, but hover is not the only way in: the click and the keyboard open + // it too, and Right is the arrow that means "into this branch". + onKeyDown={event => { + if (event.key !== "ArrowRight") return; + event.preventDefault(); + openNow(); + }} + >{renderRow({ onOpen: enter, onClick: hold })}</div> + {open && <BranchPanel panelRef={panel} row={row} width={width} label={label} + {...(align === undefined ? {} : { align })} {...(flip === undefined ? {} : { flip })} {...(side === undefined ? {} : { side })} + {...(attributes === undefined ? {} : { attributes })} + onHold={hold} onLeave={leave} onClose={close}>{children}</BranchPanel>} + </>; +} + +export type { Box }; diff --git a/packages/ui/src/components/simulator-view.tsx b/packages/ui/src/components/simulator-view.tsx index 8c58a87b..e28c8309 100644 --- a/packages/ui/src/components/simulator-view.tsx +++ b/packages/ui/src/components/simulator-view.tsx @@ -6,6 +6,7 @@ import { DevicePicker } from "./device-picker"; import { DeviceSettings } from "./device-settings"; import { bindScreenAnnotations, ScreenAnnotationsStore } from "../screen-annotations"; import { ScreenAnnotationOverlay, ScreenSelectButton } from "./screen-annotations"; +import { TestRecorderOverlay } from "./test-recorder-overlay"; const deviceButtons: { action: string; icon: LucideIcon; label: string }[] = [ { action: "home", icon: HouseIcon, label: "Home" }, @@ -36,6 +37,7 @@ export const SimulatorView = memo(function SimulatorView({ platform, visible }: </div> <div data-element="stopped" className="absolute inset-0 z-10 bg-background/70" hidden><Empty><EmptyHeader><EmptyMedia variant="icon"><PowerIcon /></EmptyMedia><EmptyTitle>Device stopped</EmptyTitle></EmptyHeader><EmptyContent><Button data-element="start-device" size="sm">Start device</Button></EmptyContent></Empty></div> <ScreenAnnotationOverlay store={annotations} /> + <TestRecorderOverlay store={annotations} /> </section> <footer className={sidebar ? "absolute bottom-2 right-2 z-30 flex items-center text-[10px] text-muted-foreground" : "flex h-11 shrink-0 items-center gap-x-3 border-t px-2 text-[10px] text-muted-foreground"}> <span data-element="notice" className="sr-only" hidden><span data-element="notice-message" /></span> diff --git a/packages/ui/src/components/step-commands.tsx b/packages/ui/src/components/step-commands.tsx new file mode 100644 index 00000000..a7430a6d --- /dev/null +++ b/packages/ui/src/components/step-commands.tsx @@ -0,0 +1,430 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { ArrowDownIcon, ArrowDownToLineIcon, ArrowLeftIcon, ArrowRightIcon, ArrowUpIcon, CheckIcon, ChevronRightIcon, EyeIcon, EyeOffIcon, HandIcon, MousePointerClickIcon, PlusIcon, SparklesIcon, TextCursorInputIcon, TypeIcon, WandSparklesIcon, XIcon } from "lucide-react"; +import { DEFAULT_ASSERTION_TIMEOUT_MS, stepIsAssertion, stepNeedsValue } from "@mobile-dev/shared/test-recording.ts"; +import type { AgentParams, AgentStepKind, PinnedStepKind, RecordedStepKind, StepOffer, SwipeDirection, VisionMode } from "@mobile-dev/shared/test-recording.ts"; +import { Branch } from "./cascading-menu"; +import { Button } from "./ui/button"; + +/** + * The command vocabulary, shared by everything that offers a step. + * + * The mirror's element menu and the step list's own "Change command" both read from here, so a + * command reads the same wherever it is chosen — and a kind the runner gains is named once. + */ +export const OFFER_LABEL: Record<RecordedStepKind, string> = { + tap: "Tap", + doubleTap: "Double tap", + longPress: "Long press", + fill: "Enter text", + scrollUntilVisible: "Scroll into view", + assertVisible: "Is Visible", + assertNotVisible: "Is Not Visible", + assertText: "Text Is", + assertContainsText: "Text Contains", + assertValue: "Field Value Is", + assertEnabled: "Is Enabled", + assertDisabled: "Is Disabled", + assertSelected: "Is Selected", + swipe: "Swipe", + agentAct: "Agent action", + agentAssert: "Agent check", + agentWaitFor: "Agent wait", +}; + +export function OfferIcon({ kind }: { kind: RecordedStepKind }) { + const Icon = kind === "agentAct" ? WandSparklesIcon + : kind === "agentAssert" || kind === "agentWaitFor" ? SparklesIcon + : kind === "swipe" ? HandIcon + : kind === "fill" ? TextCursorInputIcon + : kind === "assertText" || kind === "assertContainsText" || kind === "assertValue" ? TypeIcon + : kind === "assertNotVisible" ? EyeOffIcon + : stepIsAssertion(kind) ? EyeIcon + : kind === "scrollUntilVisible" ? ArrowDownToLineIcon + : MousePointerClickIcon; + return <Icon className="size-4 shrink-0 text-muted-foreground transition-colors group-hover/offer:text-accent-foreground group-focus-visible/offer:text-accent-foreground" />; +} + +/** One row of a menu: an action to record, or the row a submenu hangs off. */ +export function MenuRow({ children, title, onClick, onOpen, chevron, icon, attributes, disabled }: { + children: React.ReactNode; title?: string; onClick: () => void; onOpen?: () => void; + chevron?: boolean; icon?: React.ReactNode; attributes?: Record<string, string>; disabled?: boolean; +}) { + return <button + type="button" role="menuitem" title={title} disabled={disabled} {...attributes} + className="group/offer flex min-w-0 items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm transition-colors hover:bg-accent hover:text-accent-foreground focus-visible:bg-accent focus-visible:text-accent-foreground focus-visible:outline-none disabled:cursor-not-allowed disabled:opacity-50 disabled:hover:bg-transparent disabled:hover:text-inherit" + onClick={onClick} + // A submenu opens by pointing at its row, as a menu's submenu does; the click and the keyboard + // focus open it too, so it is not reachable by hover alone. + onMouseEnter={onOpen} + onFocus={onOpen} + > + {icon} + <span className="min-w-0 flex-1 truncate font-medium">{children}</span> + {chevron && <ChevronRightIcon className="size-4 shrink-0 text-muted-foreground" />} + </button>; +} + +/** The heading over a group of rows, e.g. the element a selector names, or `Assertions`. */ +export function MenuGroup({ children }: { children: React.ReactNode }) { + return <p className="px-2.5 pb-0.5 pt-1.5 text-[10px] font-medium uppercase tracking-wide text-muted-foreground">{children}</p>; +} + +/** Collects a command's text and, when requested, its wait budget. */ +export function OfferValueForm({ offer, onSubmit, onCancel, initialTimeoutMs, includeTimeout = false, back = false }: { + offer: StepOffer; onSubmit: (value?: string, timeoutMs?: number) => void; onCancel: () => void; + initialTimeoutMs?: number; includeTimeout?: boolean; back?: boolean; +}) { + const input = useRef<HTMLInputElement>(null); + const timeoutInput = useRef<HTMLInputElement>(null); + useEffect(() => { (stepNeedsValue(offer.kind) ? input : timeoutInput).current?.focus(); }, [offer, includeTimeout]); + const placeholder = offer.kind === "fill" ? "Text to enter" : "Text to expect"; + return <form className="flex flex-col gap-1.5" onSubmit={event => { + event.preventDefault(); + const timeoutMs = includeTimeout ? Number(timeoutInput.current?.value) : undefined; + if (includeTimeout && (!Number.isSafeInteger(timeoutMs) || timeoutMs! <= 0)) return; + onSubmit(stepNeedsValue(offer.kind) ? input.current?.value ?? "" : undefined, timeoutMs); + }}> + {back ? <button type="button" className="flex items-center gap-1 rounded-lg px-2 py-1 text-left text-sm hover:bg-accent" + onClick={onCancel} aria-label="Back to assertion options"><ArrowLeftIcon className="size-4" />Back</button> + : <div className="flex items-center gap-2 px-2 py-1 text-sm font-medium"> + <OfferIcon kind={offer.kind} /> + <span>{OFFER_LABEL[offer.kind]}</span> + </div>} + {stepNeedsValue(offer.kind) && <div className="flex items-center gap-1.5"> + <input ref={input} type="text" defaultValue={offer.suggestedValue ?? ""} aria-label={placeholder} placeholder={placeholder} + className="min-w-0 flex-1 rounded-lg border bg-muted/40 px-2.5 py-1.5 text-sm" + onKeyDown={event => { if (event.key === "Escape") { event.stopPropagation(); onCancel(); } }} /> + </div>} + {includeTimeout && <label className="flex items-center gap-2 text-xs text-muted-foreground"> + Timeout (ms) + <input ref={timeoutInput} type="number" min="1" step="1" required + defaultValue={initialTimeoutMs ?? DEFAULT_ASSERTION_TIMEOUT_MS} aria-label="Assertion timeout in milliseconds" + className="min-w-0 flex-1 rounded-lg border bg-muted/40 px-2.5 py-1.5 text-sm text-foreground" + onKeyDown={event => { if (event.key === "Escape") { event.stopPropagation(); onCancel(); } }} /> + </label>} + <Button type="submit" size="sm" className="self-end">Add</Button> + </form>; +} + +/** + * Every command the runner can express for one element, grouped into what acts on the app and what + * only reads it. + * + * This is the whole of the "All commands" list, and it is a component rather than a snippet + * because it is offered from two places now: over the mirror, where the element is under the + * pointer, and from a recorded step's own menu, where the element is whatever that step named. + * A row carrying text asks for it here, so neither caller has to own that form. + */ +export function StepCommandList({ commands, onChoose, renderRow, initialTimeoutMs }: { + commands: readonly StepOffer[]; + onChoose: (kind: PinnedStepKind, value?: string, timeoutMs?: number) => void; + initialTimeoutMs?: number; + /** The element menu draws its own rows, so its tooltips and attributes stay as they were. */ + renderRow?: (offer: StepOffer, choose: (offer: StepOffer) => void) => React.ReactNode; +}) { + const [asking, setAsking] = useState<StepOffer | undefined>(); + const choose = (offer: StepOffer) => { + if (offer.needsValue) { setAsking(offer); return; } + onChoose(offer.kind); + }; + if (asking) return <OfferValueForm offer={asking} initialTimeoutMs={initialTimeoutMs} onCancel={() => setAsking(undefined)} + onSubmit={(value, timeoutMs) => { setAsking(undefined); onChoose(asking.kind, value, timeoutMs); }} />; + const row = (offer: StepOffer) => stepIsAssertion(offer.kind) + ? <AssertionBranch key={`${offer.kind}:${offer.source}`} offer={offer} onChoose={onChoose} initialTimeoutMs={initialTimeoutMs} /> + : renderRow + ? renderRow(offer, choose) + : <MenuRow key={`${offer.kind}:${offer.source}`} title={offer.source} icon={<OfferIcon kind={offer.kind} />} + attributes={{ "data-recorder-offer": offer.kind }} onClick={() => choose(offer)}>{OFFER_LABEL[offer.kind]}</MenuRow>; + // The four swipes are one command chosen four ways, so they are one row with a branch rather + // than four rows that read as four commands. Every other offer is a row of its own. + const swipes = commands.filter(offer => offer.kind === "swipe"); + const actions = commands.filter(offer => !stepIsAssertion(offer.kind) && offer.kind !== "swipe"); + const assertions = commands.filter(offer => stepIsAssertion(offer.kind)); + return <> + {(actions.length > 0 || swipes.length > 0) && <MenuGroup>Actions</MenuGroup>} + {actions.map(row)} + {swipes.length > 0 && <SwipeBranch swipes={swipes} onChoose={onChoose} />} + {assertions.length > 0 && <MenuGroup>Assertions</MenuGroup>} + {assertions.map(row)} + </>; +} + +/** Assertion choices stay beside their command; only Add timeout opens an editor. */ +export function AssertionBranch({ offer, onChoose, initialTimeoutMs }: { + offer: StepOffer; + onChoose: (kind: PinnedStepKind, value?: string, timeoutMs?: number) => void; + initialTimeoutMs?: number; +}) { + const [editor, setEditor] = useState<"default" | "timeout" | undefined>(); + const selectDefault = () => { + if (stepNeedsValue(offer.kind)) { setEditor("default"); return; } + onChoose(offer.kind); + }; + return <Branch label={`${OFFER_LABEL[offer.kind]} options`} width={220} align="top" flip + attributes={{ "data-recorder-assertion-panel": offer.kind }} + renderRow={handlers => <div className="group/offer flex min-w-0 items-center rounded-lg transition-colors hover:bg-accent hover:text-accent-foreground focus-within:bg-accent focus-within:text-accent-foreground"> + <button type="button" role="menuitem" title={offer.source} + data-recorder-offer={offer.kind} + className="flex min-w-0 flex-1 items-center gap-2 rounded-lg px-2.5 py-2 text-left text-sm focus-visible:outline-none" + onClick={() => { if (stepNeedsValue(offer.kind)) handlers.onClick(); selectDefault(); }}> + <OfferIcon kind={offer.kind} /> + <span className="min-w-0 flex-1 truncate font-medium">{OFFER_LABEL[offer.kind]}</span> + </button> + <button type="button" aria-label={`${OFFER_LABEL[offer.kind]} options`} + className="flex self-stretch items-center rounded-lg px-2.5 text-muted-foreground focus-visible:outline-none" + onFocus={handlers.onOpen} onClick={handlers.onClick}> + <ChevronRightIcon className="size-4" /> + </button> + </div>} + > + {editor ? <OfferValueForm offer={offer} includeTimeout={editor === "timeout"} back + initialTimeoutMs={initialTimeoutMs} + onCancel={() => setEditor(undefined)} + onSubmit={(value, timeoutMs) => onChoose(offer.kind, value, timeoutMs)} /> + : <> + <MenuRow attributes={{ "data-recorder-assertion-default": offer.kind }} onClick={selectDefault}>Insert</MenuRow> + <MenuRow attributes={{ "data-recorder-assertion-timeout": offer.kind }} onClick={() => setEditor("timeout")}>Add timeout</MenuRow> + </>} + </Branch>; +} + +const DIRECTION_ICON = { up: ArrowUpIcon, down: ArrowDownIcon, left: ArrowLeftIcon, right: ArrowRightIcon }; + +/** + * The Swipe row and the four ways it can go. + * + * Each direction carries its own emitted code as a tooltip, which is where the two forms of the + * command show themselves: scoped to the element the step names, or `screen.swipe` over the + * viewport when the tap resolved to nothing nameable. That second form is why swipe is offered on + * a bare point at all — it is about the viewport, not about anything in it. + */ +function SwipeBranch({ swipes, onChoose }: { + swipes: readonly StepOffer[]; onChoose: (kind: PinnedStepKind, value?: string) => void; +}) { + return <Branch + label="Swipe" width={BRANCH_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-swipe" }} + renderRow={handlers => <MenuRow + chevron icon={<OfferIcon kind="swipe" />} + attributes={{ "data-recorder-panel": "swipe" }} + title={swipes[0]?.source.replace(/"(up|down|left|right)"/, "…")} + onOpen={handlers.onOpen} onClick={handlers.onClick} + >{OFFER_LABEL.swipe}</MenuRow>} + > + {swipes.map(offer => { + const direction = (offer.suggestedValue ?? "up") as SwipeDirection; + const Icon = DIRECTION_ICON[direction]; + return <MenuRow + key={direction} title={offer.source} + attributes={{ "data-recorder-swipe": direction }} + icon={<Icon className="size-4 shrink-0 text-muted-foreground" />} + onClick={() => onChoose("swipe", direction)} + >{direction[0]!.toUpperCase()}{direction.slice(1)}</MenuRow>; + })} + </Branch>; +} + +/** What `agent.assert`'s judge is shown, as three plain choices over `VisionMode`. */ +const VISION_CHOICES: { value: VisionMode; label: string; hint: string }[] = [ + { value: false, label: "Off", hint: "Judge from the element tree alone. The runner's default." }, + { value: true, label: "On", hint: "Judge from the tree and a masked screenshot." }, + // 'only' is the one that can be refused outright: with no tree to fall back on, a screen the + // runner cannot prove it redacted is denied with POLICY_DENIED rather than sent. + { value: "only", label: "Screenshot only", hint: "Judge from the screenshot alone. Denied when the pixels cannot be proven redacted." }, +]; + +const visionLabel = (value: VisionMode) => VISION_CHOICES.find(choice => choice.value === value)!.label; + +/** How wide the agentic branches are: a list of two words, then forms that hold a sentence. */ +const BRANCH_WIDTH = 190; +const FORM_WIDTH = 270; + +/** One param the user is editing. Held as rows, not as the record, so a blank row can exist. */ +type ParamRow = { id: string; key: string; value: string }; + +/** + * The `params` editor: the values an `act` instruction refers to, as key and text rows. + * + * Text rows and nothing else. `AgentParams` admits numbers, booleans, nested objects, `Secret` and + * `unique()`, but a type picker beside every row would cost more attention than it saves for the + * common case, which is a handful of scalars — so the text decides the literal when the file is + * written (see `paramValueSource`), and the shapes that carry a runtime meaning of their own stay + * something the user writes in the file. + */ +function ParamsEditor({ rows, onChange }: { rows: ParamRow[]; onChange: (rows: ParamRow[]) => void }) { + const prefix = useId(); + const set = (id: string, field: "key" | "value", next: string) => + onChange(rows.map(row => row.id === id ? { ...row, [field]: next } : row)); + return <div className="flex flex-col gap-1"> + <MenuGroup>Params</MenuGroup> + {rows.map((row, index) => <div key={row.id} className="flex items-center gap-1" data-recorder-param={String(index)}> + <input + type="text" value={row.key} aria-label={`Param ${index + 1} name`} placeholder="name" + data-recorder-param-key={String(index)} + className="w-20 min-w-0 shrink-0 rounded-lg border bg-muted/40 px-2 py-1 text-xs" + onChange={event => set(row.id, "key", event.target.value)} /> + <input + type="text" value={row.value} aria-label={`Param ${index + 1} value`} placeholder="value" + data-recorder-param-value={String(index)} + className="min-w-0 flex-1 rounded-lg border bg-muted/40 px-2 py-1 text-xs" + onChange={event => set(row.id, "value", event.target.value)} /> + <Button type="button" variant="ghost" size="icon-xs" className="shrink-0" + aria-label={`Remove param ${index + 1}`} data-recorder-param-remove={String(index)} + onClick={() => onChange(rows.filter(entry => entry.id !== row.id))}><XIcon /></Button> + </div>)} + <MenuRow icon={<PlusIcon className="size-4 shrink-0 text-muted-foreground" />} + attributes={{ "data-recorder-param-add": "" }} + title="A value the instruction refers to, passed to the agent rather than written into it" + onClick={() => onChange([...rows, { id: `${prefix}-${rows.length}-${Date.now()}`, key: "", value: "" }])} + >Add param</MenuRow> + </div>; +} + +/** + * The rows as the step records them: in order, blank keys dropped, the last of a repeated key + * winning — which is the only thing an object literal can mean. See `AgentParams`. + */ +function toParams(rows: ParamRow[]): AgentParams | undefined { + const params: AgentParams = {}; + for (const row of rows) if (row.key.trim() !== "") params[row.key.trim()] = row.value; + return Object.keys(params).length > 0 ? params : undefined; +} + +/** The sentence field both agent forms are built around. */ +function InstructionField({ kind, value, onChange }: { + kind: AgentStepKind; value: string; onChange: (value: string) => void; +}) { + const input = useRef<HTMLInputElement>(null); + useEffect(() => { input.current?.focus(); }, []); + return <input + ref={input} type="text" value={value} data-recorder-agentic-input={kind} + aria-label={kind === "agentAct" ? "What the agent should do" : "What should be true"} + placeholder={kind === "agentAct" ? 'tap the "Explore" tab' : "the cart shows two items"} + className="min-w-0 rounded-lg border bg-muted/40 px-2.5 py-1.5 text-sm" + onChange={event => onChange(event.target.value)} />; +} + +function FormHeading({ kind }: { kind: AgentStepKind }) { + return <div className="flex items-center gap-2 px-2 py-1 text-sm font-medium"> + <OfferIcon kind={kind} /> + <span>{OFFER_LABEL[kind]}</span> + </div>; +} + +/** + * The Agentic branch: Act and Assert, each opening its own form beside it. + * + * Neither call takes a locator — `agent.act(instruction)` and `agent.assert(assertion)` are a + * sentence and nothing else — so the element the user right-clicked only writes the first draft of + * that sentence and is then forgotten. That is why this is offered where nothing can be named at + * all: describing the thing is the step, and a coordinate is not. + */ +export function AgenticBranch({ suggestion, onRecord }: { + /** The sentence the Act field starts from; see `agentActSuggestion`. */ + suggestion: string; + onRecord: (step: + | { kind: "agentAct"; instruction: string; params?: AgentParams } + | { kind: "agentAssert" | "agentWaitFor"; instruction: string; vision?: VisionMode }) => void; +}) { + return <Branch + label="Agentic" width={BRANCH_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-agentic" }} + renderRow={handlers => <MenuRow + chevron icon={<OfferIcon kind="agentAct" />} + attributes={{ "data-recorder-panel": "agentic" }} + title="Describe the step and let the agent resolve it at run time" + onOpen={handlers.onOpen} onClick={handlers.onClick} + >Agentic</MenuRow>} + > + <Branch + label="Act" width={FORM_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-agentic-act" }} + renderRow={handlers => <MenuRow + chevron icon={<OfferIcon kind="agentAct" />} + attributes={{ "data-recorder-agentic": "agentAct" }} + title="agent.act(…) — the agent carries out an instruction" + onOpen={handlers.onOpen} onClick={handlers.onClick} + >Act</MenuRow>} + ><ActForm suggestion={suggestion} onRecord={onRecord} /></Branch> + <Branch + label="Assert" width={FORM_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-agentic-assert" }} + renderRow={handlers => <MenuRow + chevron icon={<OfferIcon kind="agentAssert" />} + attributes={{ "data-recorder-agentic": "agentAssert" }} + title="agent.assert(…) — the agent judges whether something holds" + onOpen={handlers.onOpen} onClick={handlers.onClick} + >Assert</MenuRow>} + ><JudgementForm kind="agentAssert" onRecord={onRecord} /></Branch> + <Branch + label="Wait for" width={FORM_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-agentic-waitfor" }} + renderRow={handlers => <MenuRow + chevron icon={<OfferIcon kind="agentWaitFor" />} + attributes={{ "data-recorder-agentic": "agentWaitFor" }} + title="agent.waitFor(…) — the agent judges on a loop until it holds" + onOpen={handlers.onOpen} onClick={handlers.onClick} + >Wait for</MenuRow>} + ><JudgementForm kind="agentWaitFor" onRecord={onRecord} /></Branch> + </Branch>; +} + +function ActForm({ suggestion, onRecord }: { + suggestion: string; + onRecord: (step: { kind: "agentAct"; instruction: string; params?: AgentParams }) => void; +}) { + const [instruction, setInstruction] = useState(suggestion); + const [rows, setRows] = useState<ParamRow[]>([]); + const params = toParams(rows); + return <form className="flex flex-col gap-1.5" onSubmit={event => { + event.preventDefault(); + // No params means no options object at all, so the line stays `agent.act("…")`. + onRecord({ kind: "agentAct", instruction, ...(params ? { params } : {}) }); + }}> + <FormHeading kind="agentAct" /> + <InstructionField kind="agentAct" value={instruction} onChange={setInstruction} /> + <ParamsEditor rows={rows} onChange={setRows} /> + <div className="flex justify-end"><Button type="submit" size="sm" className="shrink-0">Add</Button></div> + </form>; +} + +/** + * The form behind both judgements. + * + * `assert` judges once and `waitFor` judges on a loop, but what the user supplies is the same + * sentence and the same choice of what the judge is shown, so they share a form. `waitFor`'s own + * `interval` and `maxModelCalls` are budgets rather than a description of the condition, and like + * `act`'s they stay something the user sets in the file. + */ +function JudgementForm({ kind, onRecord }: { + kind: "agentAssert" | "agentWaitFor"; + onRecord: (step: { kind: "agentAssert" | "agentWaitFor"; instruction: string; vision?: VisionMode }) => void; +}) { + const [instruction, setInstruction] = useState(""); + const [vision, setVision] = useState<VisionMode>(false); + return <form className="flex flex-col gap-1.5" onSubmit={event => { + event.preventDefault(); + onRecord({ kind, instruction, ...(vision ? { vision } : {}) }); + }}> + <FormHeading kind={kind} /> + <InstructionField kind={kind} value={instruction} onChange={setInstruction} /> + <Branch + label="Vision" width={BRANCH_WIDTH} align="top" flip + attributes={{ "data-element": "recorder-vision" }} + renderRow={handlers => <MenuRow + chevron icon={<EyeIcon className="size-4 shrink-0 text-muted-foreground" />} + attributes={{ "data-recorder-panel": "vision" }} + title="What the judge is shown" + onOpen={handlers.onOpen} onClick={handlers.onClick} + >Vision: {visionLabel(vision)}</MenuRow>} + > + {VISION_CHOICES.map(choice => <MenuRow + key={String(choice.value)} title={choice.hint} + attributes={{ "data-recorder-vision": String(choice.value) }} + icon={<CheckIcon className={`size-4 shrink-0 ${choice.value === vision ? "text-foreground" : "opacity-0"}`} />} + onClick={() => setVision(choice.value)} + >{choice.label}</MenuRow>)} + </Branch> + <div className="flex justify-end"><Button type="submit" size="sm" className="shrink-0">Add</Button></div> + </form>; +} diff --git a/packages/ui/src/components/test-recorder-overlay.tsx b/packages/ui/src/components/test-recorder-overlay.tsx new file mode 100644 index 00000000..635dafad --- /dev/null +++ b/packages/ui/src/components/test-recorder-overlay.tsx @@ -0,0 +1,253 @@ +import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react"; +import { createPortal } from "react-dom"; +import { ListIcon, LoaderCircleIcon } from "lucide-react"; +import type { ScreenAnnotationsStore } from "../screen-annotations"; +import { testRecorder } from "../test-recorder"; +import { agentActSuggestion, stepIsAssertion } from "@mobile-dev/shared/test-recording.ts"; +import type { StepOffer, StepTarget } from "@mobile-dev/shared/test-recording.ts"; +import { AgenticBranch, AssertionBranch, MenuRow, OFFER_LABEL, OfferIcon, OfferValueForm, StepCommandList } from "./step-commands"; +import { usePlacement, windowView as view } from "./cascading-menu"; +import { MARGIN, placeBelow, placeBeside } from "../popover-placement"; +import type { Box } from "../popover-placement"; + +/** + * The element menu: what the runner can do with the element the outline is on. + * + * The selector is named once, then every row is a plain-language action the `e2e` runner can + * express for it. The emitted code remains available as a tooltip and appears in the test preview + * after the action is chosen. A row that needs a value — `fill` types it, the text matchers + * compare with it — asks for one here instead of recording an empty string literal. + * + * The short list is what most steps are. Everything else the runner can express hangs off "All + * commands" as a submenu of its own, opened by pointing at the row. Both are portals fitted to the + * window: they may lie over the mirror, the toolbar or the panel's edge, and they are never cut + * off by the stage that the element they are about happens to live in. + */ +function ElementMenu({ label, offers, commands, matches, element, width, anchor }: { + label: string; offers: readonly StepOffer[]; commands: readonly StepOffer[]; + matches: number; element?: StepTarget; width: number; anchor: () => Box | undefined; +}) { + const [asking, setAsking] = useState<StepOffer | undefined>(); + const [open, setOpen] = useState(false); + const menu = useRef<HTMLDivElement>(null); + const row = useRef<HTMLDivElement>(null); + const target = label.replace(/^Tap /, ""); + const placement = usePlacement(menu, () => { + const rect = anchor(); + return rect && placeBelow(rect, { width, height: menu.current?.offsetHeight ?? 0 }, view()); + }, [anchor, width, asking]); + const choose = (offer: StepOffer) => { + if (offer.needsValue) { setAsking(offer); return; } + void testRecorder.chooseOffer(offer.kind); + }; + const offerRow = (offer: StepOffer) => <MenuRow + key={`${offer.kind}:${offer.source}`} + title={offer.source} + icon={<OfferIcon kind={offer.kind} />} + attributes={{ "data-recorder-offer": offer.kind }} + onClick={() => choose(offer)} + >{OFFER_LABEL[offer.kind]}</MenuRow>; + return createPortal(<div + ref={menu} + data-element="recorder-menu" role="menu" aria-label="Element steps" + className="pointer-events-auto fixed z-50 flex flex-col gap-1.5 overflow-y-auto rounded-xl border bg-popover p-1.5 text-popover-foreground shadow-lg [&_button:not(:disabled)]:cursor-pointer" + // Hidden until it has been measured, so it is never seen at the position it was measured from. + style={{ left: placement?.left ?? 0, top: placement?.top ?? 0, width, maxHeight: placement?.maxHeight, visibility: placement ? "visible" : "hidden" }} + onPointerDown={event => event.stopPropagation()} + onContextMenu={event => event.preventDefault()} + > + <div className="min-w-0 px-2.5 py-1.5"> + <p className="text-[10px] font-medium uppercase tracking-wide text-muted-foreground">Selected element</p> + <p className="truncate text-sm font-medium" title={target}>{target}</p> + {matches > 1 && <p className="mt-1 text-[11px] leading-4 text-amber-600 dark:text-amber-500"> + {matches} matches · using this one by position. Add a testID for stability. + </p>} + </div> + <div className="-mx-1.5 border-t" /> + {asking + ? <OfferValueForm offer={asking} onCancel={() => setAsking(undefined)} + onSubmit={(value, timeoutMs) => { setAsking(undefined); void testRecorder.chooseOffer(asking.kind, value, timeoutMs); }} /> + : <> + {/* A coordinate gets the tap and nothing else, and silence about the other three rows reads + as a broken menu. The step list flags the same thing once the step is recorded. */} + {offers.length === 1 && offers[0]?.kind === "tap" && offers[0].source.startsWith("screen.tapAt") + && <p className="px-2.5 pt-1 text-[11px] text-amber-600 dark:text-amber-500"> + Nothing here can be named. The coordinate can still be tapped or swiped, or describe the step under Agentic. + </p>} + {offers.map(offer => stepIsAssertion(offer.kind) + ? <AssertionBranch key={`${offer.kind}:${offer.source}`} offer={offer} + onChoose={(kind, value, timeoutMs) => void testRecorder.chooseOffer(kind, value, timeoutMs)} /> + : offerRow(offer))} + <div className="-mx-1.5 border-t" /> + <div ref={row} className="flex flex-col" onMouseEnter={() => setOpen(true)}> + <MenuRow + chevron + icon={<ListIcon className="size-4 shrink-0 text-muted-foreground transition-colors group-hover/offer:text-accent-foreground" />} + attributes={{ "data-recorder-panel": "commands" }} + onOpen={() => setOpen(true)} + onClick={() => setOpen(true)} + >All commands</MenuRow> + </div> + {open && <CommandMenu + commands={commands} + // Narrower than the menu it hangs off: its rows are short, and every point it does not + // take is a point of the menu behind it that a tight window leaves visible. + width={Math.min(width, 220)} + row={row} + offerRow={offerRow} + onClose={() => setOpen(false)} + />} + {/* The one branch that needs no selector, which is why it is here even when the element + menu has nothing but a coordinate to offer. */} + <AgenticBranch suggestion={agentActSuggestion(element)} onRecord={step => testRecorder.recordAgentStep(step)} /> + </>} + </div>, document.body); +} + +/** + * The "All commands" submenu: a popover of its own, beside the row it hangs off. + * + * It always opens to the right, the side its chevron points at — see `placeBeside` for why there + * is no flip to the left — and its bottom lines up with its row, growing upward since the row is + * the last in the menu. Both are pushed back in if they would take it past an edge of the window. + * + * It is a portal rather than a child of the menu for two reasons: the menu scrolls once it is + * taller than the window, and a scrolling box clips what hangs out of it; and the submenu has to + * be free to lie outside the menu, the stage and the panel alike. + */ +function CommandMenu({ commands, width, row, offerRow, onClose }: { + commands: readonly StepOffer[]; width: number; row: React.RefObject<HTMLDivElement | null>; + offerRow: (offer: StepOffer) => React.ReactNode; onClose: () => void; +}) { + const panel = useRef<HTMLDivElement>(null); + const placement = usePlacement(panel, () => { + const anchor = row.current?.getBoundingClientRect(); + return anchor && placeBeside(anchor, { width, height: panel.current?.offsetHeight ?? 0 }, view()); + }, [row, width, commands]); + + // The row and this panel are one hover target with a gap between them, so leaving either one + // only starts a close that entering the other cancels — otherwise crossing the gap closes it. + const leaving = useRef<ReturnType<typeof setTimeout>>(undefined); + const hold = () => { if (leaving.current) clearTimeout(leaving.current); }; + const release = useCallback(() => { + hold(); + const closeWhenBranchesLeave = () => { + // Assertion choices are portalled beside this menu. Keep their parent mounted while the + // pointer crosses into them, then close once the branch itself has gone. + if (document.querySelector("[data-recorder-assertion-panel]")) { + leaving.current = setTimeout(closeWhenBranchesLeave, 120); + } else onClose(); + }; + leaving.current = setTimeout(closeWhenBranchesLeave, 120); + }, [onClose]); + useEffect(() => { + const node = row.current; + node?.addEventListener("mouseleave", release); + node?.addEventListener("mouseenter", hold); + return () => { hold(); node?.removeEventListener("mouseleave", release); node?.removeEventListener("mouseenter", hold); }; + }, [row, release]); + return createPortal(<div + ref={panel} + data-element="recorder-commands" role="menu" aria-label="All commands" + className="pointer-events-auto fixed z-50 flex flex-col gap-1.5 overflow-y-auto rounded-xl border bg-popover p-1.5 text-popover-foreground shadow-lg [&_button:not(:disabled)]:cursor-pointer" + style={{ left: placement?.left ?? 0, top: placement?.top ?? 0, width, maxHeight: placement?.maxHeight, visibility: placement ? "visible" : "hidden" }} + onMouseEnter={hold} + onMouseLeave={release} + onPointerDown={event => event.stopPropagation()} + onContextMenu={event => event.preventDefault()} + > + {/* The rows are the element menu's own, so a command reads and behaves the same in both. */} + <StepCommandList commands={commands} onChoose={(kind, value, timeoutMs) => void testRecorder.chooseOffer(kind, value, timeoutMs)} renderRow={offer => offerRow(offer)} /> + </div>, document.body); +} + +/** + * The outline over the mirror while the Testing panel records: the element a tap would name, the + * step it would add, and — on a right-click — the menu of every step that element supports. + * + * It borrows the annotation store's viewport — the canvas rectangle inside the stage, which the + * simulator panel keeps current through every resize and bezel change — rather than measuring the + * canvas again. The bounds themselves come from the recorder, in the same screen space it resolved + * them in, so the outline cannot drift from what gets recorded, wherever in the workspace the + * mirror sits. + */ +export function TestRecorderOverlay({ store }: { store: ScreenAnnotationsStore }) { + const { viewport: v } = useSyncExternalStore(store.subscribe, store.getSnapshot); + const { hover, menu, preparing } = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + // State rather than a ref: the menu is a child, and a child's layout effect runs before the + // parent's ref is attached, so a ref here is still null the first time the menu measures itself. + const [host, setHost] = useState<HTMLDivElement | null>(null); + // The outline and the menu are drawn from whichever is there. A hover is dropped by anything that + // invalidates the tree — an input, a frame, a slow read — and the menu, which the user asked for + // explicitly, carries its own rectangle so it cannot be taken away with one. + const anchor = hover ?? menu; + const bounds = menu?.bounds; + const screen = menu?.screen; + // The menu is placed against the window, so it needs the element's rectangle in the window's own + // coordinates: the stage's position plus the element's position inside it. It is read when the + // menu measures itself rather than passed as numbers, so a panel that moves under an open menu — + // a resize, a scroll, a pane dragged wider — moves the menu with it. + const menuAnchor = useCallback(() => { + const stage = host?.getBoundingClientRect(); + if (!stage || !bounds || !screen || !v.width || !v.height) return undefined; + const left = stage.left + v.x + bounds.x / screen.width * v.width; + const top = stage.top + v.y + bounds.y / screen.height * v.height; + const width = bounds.width / screen.width * v.width; + const height = bounds.height / screen.height * v.height; + return { left, top, width, height, right: left + width, bottom: top + height }; + }, [host, bounds, screen, v]); + // The canvas closes the menu when it is clicked through, but the menu is no longer inside the + // stage and the rest of the panel is not the canvas: a click anywhere else closes it too. + useEffect(() => { + if (!menu) return; + const dismiss = (event: PointerEvent) => { + const target = event.target as Element | null; + if (target?.closest("[data-element=recorder-menu],[data-element=recorder-commands]")) return; + testRecorder.closeMenu(); + }; + window.addEventListener("pointerdown", dismiss); + return () => window.removeEventListener("pointerdown", dismiss); + }, [menu]); + if (!v.width || !v.height) return null; + // Record arms the recorder at once, but nothing on the mirror can be selected until the runner's + // element tree has been read, and that takes a second or two. Without this the panel said + // "Stop" while the first few clicks did nothing, which reads as a broken recorder rather than as + // a wait. The overlay covers the canvas — not the whole stage — so it says *where* the wait is, + // and it lets clicks through: the mirror still drives the app while the tree is being read. + if (preparing) return <div className="pointer-events-none absolute inset-0 z-20" data-element="recorder-preparing"> + <div className="absolute flex items-center justify-center bg-background/60 backdrop-blur-[1px]" + style={{ left: v.x, top: v.y, width: v.width, height: v.height }} role="status"> + <span className="flex items-center gap-2 rounded-md bg-background/90 px-3 py-1.5 text-xs font-medium shadow-sm"> + <LoaderCircleIcon className="size-3.5 animate-spin" />preparing... + </span> + </div> + </div>; + if (!anchor) return null; + const rect = { + left: v.x + anchor.bounds.x / anchor.screen.width * v.width, + top: v.y + anchor.bounds.y / anchor.screen.height * v.height, + width: anchor.bounds.width / anchor.screen.width * v.width, + height: anchor.bounds.height / anchor.screen.height * v.height, + }; + // Above the outline when there is room, inside its top edge when it is against the stage top. + const above = rect.top > v.y + 22; + return <div ref={setHost} className="pointer-events-none absolute inset-0 z-20" data-element="recorder-hover"> + <div className={`absolute border-2 border-solid ${anchor.ambiguous ? "border-amber-500 bg-amber-500/10" : "border-blue-500 bg-blue-500/10"}`} style={rect} /> + <span + className={`absolute max-w-[min(18rem,100%)] truncate rounded px-1.5 py-0.5 text-[10px] font-medium text-white shadow-sm ${anchor.ambiguous ? "bg-amber-600" : "bg-blue-600"}`} + style={{ left: Math.max(4, Math.min(rect.left, v.stageWidth - 16)), top: Math.max(2, above ? rect.top - 20 : rect.top + 2) }} + >{anchor.label}{hover && hover.levels > 1 ? ` · ${hover.level + 1}/${hover.levels}` : ""}</span> + {/* Anchored to the element, not to the pointer, so walking out with the wheel moves the menu + with the element it is about. */} + {menu && <ElementMenu + label={menu.label} + offers={menu.offers} + commands={menu.commands} + matches={menu.matches} + element={menu.target} + // Wide enough for a row to read as a sentence, narrow enough to leave the mirror visible. + width={Math.max(180, Math.min(280, window.innerWidth - MARGIN * 2))} + anchor={menuAnchor} + />} + </div>; +} diff --git a/packages/ui/src/components/testing-view.tsx b/packages/ui/src/components/testing-view.tsx new file mode 100644 index 00000000..90d6e925 --- /dev/null +++ b/packages/ui/src/components/testing-view.tsx @@ -0,0 +1,1099 @@ +import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react"; +import { BotIcon, CheckIcon, ChevronDownIcon, ChevronRightIcon, CircleIcon, CircleXIcon, CopyIcon, FileCode2Icon, FilePlus2Icon, FlaskConicalIcon, FolderOpenIcon, GripVerticalIcon, HistoryIcon, LoaderCircleIcon, MoreHorizontalIcon, PlayIcon, SaveIcon, SquareIcon, Trash2Icon, UserIcon, XIcon } from "lucide-react"; +import type { DeviceAppsStore } from "../device-apps.ts"; +import type { TestingPanel } from "../testing-panel.ts"; +import { OPEN_STEP, formatRunDuration, testRecorder } from "../test-recorder.ts"; +import type { DevicePlatform, TestRunView } from "../test-recorder.ts"; +import { isRecordableDevice, recordablePlatform } from "@mobile-dev/shared/panel-target.ts"; +import { allStepOffers, isAgentStep, testFileSlug } from "@mobile-dev/shared/test-recording.ts"; +import { draftLabel } from "@mobile-dev/shared/test-drafts.ts"; +import type { RecordedStep } from "@mobile-dev/shared/test-recording.ts"; +import { MenuRow, OFFER_LABEL, OfferIcon, StepCommandList } from "./step-commands.tsx"; +import { Branch } from "./cascading-menu.tsx"; +import { CODE_TOKEN_CLASS, changedLines, highlightCode } from "../code-highlight.ts"; +import { testSourceSuggestions } from "../test-source-suggestions.ts"; +import { changeStoredStepCommand, duplicateStoredStep, moveStoredStep, removeStoredStep, storedStepCommands, storedTestSteps, storedTestUntilSource } from "../stored-test-steps.ts"; +import { Alert, AlertDescription, AlertTitle } from "./ui/alert.tsx"; +import { Button } from "./ui/button.tsx"; +import { Popover, PopoverContent, PopoverTrigger } from "./ui/popover.tsx"; +import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "./ui/resizable.tsx"; +import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectValue } from "./ui/base-select.tsx"; + +/** + * The project selector: folders the panel has recorded against, plus the system folder picker for + * any other one. The panel holds the choice and the server persists it. + */ +function TestingProjectPicker({ panel }: { panel: TestingPanel }) { + const state = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const [open, setOpen] = useState(false); + const selected = state.projects.find(project => project.path === state.projectPath); + const label = state.projectPath + ? selected?.name ?? state.projectPath.split("/").filter(Boolean).at(-1) ?? state.projectPath + : "Choose a project"; + const choose = (projectPath: string) => { setOpen(false); void panel.chooseProject(projectPath); }; + return <div className="flex min-w-0 items-center gap-1"> + <Popover open={open} onOpenChange={value => { setOpen(value); if (value) void panel.refreshProjects(); }}> + <PopoverTrigger asChild> + <Button type="button" variant="outline" size="sm" className="min-w-0 max-w-72 justify-between gap-1.5 font-normal" aria-label="Testing project" title={state.projectPath || "Choose an e2e project"}> + <span className="min-w-0 truncate">{label}</span> + <ChevronDownIcon className="shrink-0 transition-transform" style={{ transform: open ? "rotate(180deg)" : undefined }} /> + </Button> + </PopoverTrigger> + <PopoverContent align="start" className="flex max-h-(--radix-popover-content-available-height) w-max min-w-[min(var(--radix-popover-trigger-width),var(--radix-popover-content-available-width))] max-w-(--radix-popover-content-available-width) flex-col gap-1 overflow-y-auto p-1 [&_button:not(:disabled)]:cursor-pointer" aria-label="Testing project"> + {state.projects.map(project => <Button key={project.path} type="button" variant="ghost" data-testing-project-option={project.path} aria-pressed={project.path === state.projectPath} + className="h-auto min-h-7 min-w-0 flex-1 justify-start rounded-md px-1.5 py-1 text-left font-normal" onClick={() => choose(project.path)}> + <span className="flex min-w-0 items-center gap-2"> + <CheckIcon className={`size-3.5 shrink-0 ${project.path === state.projectPath ? "opacity-100" : "opacity-0"}`} /> + <span className="flex min-w-0 flex-col"> + <span className="truncate">{project.name}</span> + <span className="truncate text-xs text-muted-foreground">{project.path}</span> + </span> + </span> + </Button>)} + {state.projects.length === 0 && <p className="px-1.5 py-1 text-xs text-muted-foreground">No projects yet. Choose a folder holding an e2e.config.ts.</p>} + <Button type="button" variant="ghost" data-testing-project-browse disabled={state.pickingProject} className="h-auto min-h-7 justify-start rounded-md px-1.5 py-1 text-left font-normal" onClick={() => { setOpen(false); void panel.pickProject(); }}> + <FolderOpenIcon />Choose another folder… + </Button> + </PopoverContent> + </Popover> + {state.pickingProject && <span className="flex shrink-0 items-center gap-1 text-xs text-muted-foreground" role="status"><LoaderCircleIcon className="animate-spin" />Choose a folder…</span>} + </div>; +} + + +/** How long ago a draft was last touched, short enough to sit under its name in a row. */ +function sinceLabel(iso: string): string { + const at = Date.parse(iso); + if (Number.isNaN(at)) return ""; + const seconds = Math.max(0, Math.round((Date.now() - at) / 1000)); + if (seconds < 60) return "just now"; + const minutes = Math.round(seconds / 60); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.round(minutes / 60); + if (hours < 24) return `${hours}h ago`; + return `${Math.round(hours / 24)}d ago`; +} + +/** A group header in the tests list: a disclosure that says how many rows it is holding. */ +function ListGroup({ label, count, open, onToggle, attributes, children }: { + label: string; count: number; open: boolean; onToggle: () => void; + attributes?: Record<string, string>; children: React.ReactNode; +}) { + return <div className="flex min-w-0 flex-col"> + <button type="button" {...attributes} aria-expanded={open} + className="flex min-w-0 items-center gap-1.5 rounded-md px-1.5 py-1 text-left text-[10px] font-medium uppercase tracking-wide text-muted-foreground hover:bg-accent hover:text-accent-foreground" + onClick={onToggle}> + {open ? <ChevronDownIcon className="size-3.5 shrink-0" /> : <ChevronRightIcon className="size-3.5 shrink-0" />} + <span className="min-w-0 truncate">{label}</span> + <span className="shrink-0 tabular-nums">{count}</span> + </button> + {open && <div className="flex min-w-0 flex-col gap-1 pl-1">{children}</div>} + </div>; +} + +/** + * The delete a row asks through: the trash icon, and the question in a popover beside it. + * + * Deleting a recording throws away steps nothing can replay, and deleting a stored test removes a + * file from the user's repository; both used to happen on one click of an icon that sits beside + * the row the user was reaching for. The question hangs off the icon rather than replacing the row + * so the row it is about stays legible underneath it, and so the question has room to say what + * will go and that it cannot be undone — which a row the width of a file name does not. + */ +function ConfirmDelete({ question, label, title, disabled, busy, attributes, onConfirm }: { + /** What is about to go, as the sentence the popover opens with. */ + question: string; + /** The trash button's own accessible name, e.g. `Delete flow.e2e.ts`. */ + label: string; + title: string; + disabled?: boolean; + busy?: boolean; + attributes?: Record<string, string>; + onConfirm: () => void; +}) { + const [open, setOpen] = useState(false); + return <Popover open={open} onOpenChange={setOpen}> + <PopoverTrigger asChild> + <Button type="button" variant="ghost" size="sm" disabled={disabled || busy} {...attributes} + className="size-7 shrink-0 p-0 text-muted-foreground hover:text-destructive" + title={title} aria-label={label}> + {busy ? <LoaderCircleIcon className="animate-spin" /> : <Trash2Icon />} + </Button> + </PopoverTrigger> + <PopoverContent align="end" className="flex w-64 flex-col gap-2 p-2.5 [&_button:not(:disabled)]:cursor-pointer" + role="alertdialog" aria-label={question}> + <p className="min-w-0 break-words text-xs">{question}</p> + <div className="flex justify-end gap-2"> + <Button type="button" size="sm" variant="ghost" onClick={() => setOpen(false)}>Cancel</Button> + <Button type="button" size="sm" variant="destructive" data-confirm-delete="" + onClick={() => { setOpen(false); onConfirm(); }}>Delete</Button> + </div> + </PopoverContent> + </Popover>; +} + +/** + * The flows of this project: the ones that are not stored yet, and the tests the project holds. + * + * A flow lives in the panel document until Save writes it into the project, and the host rebuilds + * that document whenever the user looks at another tool or another chat — so without the first + * group, a flow they had not saved, or one their agent recorded while they were reading the chat, + * was simply gone when they came back. + * + * The second group is here because saving used to have the same effect for the opposite reason: + * the draft is deleted once the file exists, so the test the user had just saved left the list. + * Now it moves — out of **Not stored** and into the project's own tests, which are read from the + * folder so a test written by hand or by an agent is listed beside the recorded ones. That group + * starts collapsed: it is the archive, and what is being worked on is above it. + */ +function TestingDraftPicker({ panel }: { panel: TestingPanel }) { + const state = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const recorder = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + const [open, setOpen] = useState(false); + const [unstoredOpen, setUnstoredOpen] = useState(true); + const [storedOpen, setStoredOpen] = useState(false); + const agentRecording = state.agent?.active === true; + // The one on screen may not be in the list yet: the autosave is debounced, and the first keep of + // a brand new flow lands a moment after the first step. + if (state.drafts.length === 0 && !state.draftId && state.tests.length === 0) return null; + const current = state.drafts.find(draft => draft.id === state.draftId); + const selectedTest = state.tests.find(test => test.relativePath === state.selectedTestPath); + const label = selectedTest ? selectedTest.name : current ? draftLabel(current) : recorder.steps.length > 0 ? draftLabel(recorder) : "Tests"; + return <Popover open={open} onOpenChange={value => { + setOpen(value); + if (value) { void panel.refreshDrafts(); void panel.refreshTests(); } + }}> + <PopoverTrigger asChild> + <Button type="button" variant="outline" size="sm" data-testing-drafts className="min-w-0 max-w-64 shrink-0 justify-between gap-1.5 font-normal" + aria-label="Recordings and tests" title="What has not been stored yet, and the tests this project holds"> + <span className="flex min-w-0 items-center gap-1.5"> + {state.loadingDraft ? <LoaderCircleIcon className="shrink-0 animate-spin" /> : <HistoryIcon className="shrink-0" />} + <span className="min-w-0 truncate">{label}</span> + </span> + <ChevronDownIcon className="shrink-0 transition-transform" style={{ transform: open ? "rotate(180deg)" : undefined }} /> + </Button> + </PopoverTrigger> + <PopoverContent align="start" className="flex max-h-(--radix-popover-content-available-height) w-max min-w-[min(var(--radix-popover-trigger-width),var(--radix-popover-content-available-width))] max-w-(--radix-popover-content-available-width) flex-col gap-1 overflow-y-auto p-1 [&_button:not(:disabled)]:cursor-pointer" aria-label="Recordings and tests"> + <ListGroup label="Not stored" count={state.drafts.length} open={unstoredOpen} + attributes={{ "data-testing-group": "unstored" }} onToggle={() => setUnstoredOpen(value => !value)}> + {state.drafts.map(draft => { + const selected = draft.id === state.draftId; + return <span key={draft.id} className="flex min-w-0 items-center gap-1"> + <Button type="button" variant="ghost" data-testing-draft-option={draft.id} aria-pressed={selected} disabled={agentRecording} + className="h-auto min-h-7 min-w-0 flex-1 justify-start rounded-md px-1.5 py-1 text-left font-normal" + onClick={() => { setOpen(false); void panel.openDraft(draft.id); }}> + <span className="flex min-w-0 items-center gap-2"> + <CheckIcon className={`size-3.5 shrink-0 ${selected ? "opacity-100" : "opacity-0"}`} /> + <span className="flex min-w-0 flex-col"> + <span className="truncate">{draftLabel(draft)}</span> + <span className="flex items-center gap-1 truncate text-xs text-muted-foreground"> + {draft.origin === "agent" ? <BotIcon className="size-3 shrink-0" aria-label="Recorded by an agent" /> : <UserIcon className="size-3 shrink-0" aria-label="Recorded by you" />} + {draft.stepCount} step{draft.stepCount === 1 ? "" : "s"} + {sinceLabel(draft.updatedAt) ? ` · ${sinceLabel(draft.updatedAt)}` : ""} + </span> + </span> + </span> + </Button> + <ConfirmDelete attributes={{ "data-testing-draft-delete": draft.id }} disabled={agentRecording && selected} + title={selected ? "Delete this recording and clear the panel" : "Delete this recording"} + label={`Delete ${draftLabel(draft)}`} + question={selected + ? `Delete “${draftLabel(draft)}” and clear the panel? Its ${draft.stepCount} step${draft.stepCount === 1 ? "" : "s"} cannot be brought back.` + : `Delete “${draftLabel(draft)}”? Its ${draft.stepCount} step${draft.stepCount === 1 ? "" : "s"} cannot be brought back.`} + onConfirm={() => void panel.deleteDraft(draft.id)} /> + </span>; + })} + {state.drafts.length === 0 && <p className="px-1.5 py-1 text-xs text-muted-foreground">Everything recorded here has been saved into the project.</p>} + <Button type="button" variant="ghost" data-testing-draft-new disabled={agentRecording} + className="h-auto min-h-7 justify-start rounded-md px-1.5 py-1 text-left font-normal" + title="Keep what is on screen and start an empty flow." + onClick={() => { setOpen(false); void panel.newRecording(); }}><FilePlus2Icon />New recording</Button> + </ListGroup> + + <div className="-mx-1 border-t" /> + + <ListGroup label="In the project" count={state.tests.length} open={storedOpen} + attributes={{ "data-testing-group": "stored" }} onToggle={() => setStoredOpen(value => !value)}> + {state.tests.map(stored => { + const selected = stored.relativePath === state.selectedTestPath; + return <span key={stored.relativePath} className="flex min-w-0 items-center gap-1" data-testing-test={stored.relativePath}> + <Button type="button" variant="ghost" data-testing-test-option={stored.relativePath} aria-pressed={selected} disabled={agentRecording} + className="h-auto min-h-7 min-w-0 flex-1 justify-start gap-2 rounded-md px-1.5 py-1 text-left font-normal" title={stored.relativePath} + onClick={() => { setOpen(false); void panel.openTest(stored.relativePath); }}> + <CheckIcon className={`size-3.5 shrink-0 ${selected ? "opacity-100" : "opacity-0"}`} /> + <FileCode2Icon className="size-3.5 shrink-0 text-muted-foreground" aria-hidden /> + <span className="flex min-w-0 flex-col"> + <span className="truncate text-sm">{stored.name}</span> + <span className="truncate text-xs text-muted-foreground"> + {stored.relativePath}{sinceLabel(stored.updatedAt) ? ` · ${sinceLabel(stored.updatedAt)}` : ""} + </span> + </span> + </Button> + <ConfirmDelete attributes={{ "data-testing-test-delete": stored.relativePath }} + busy={state.deletingTest === stored.relativePath} + title="Delete this test from the project" label={`Delete ${stored.name}`} + question={`Delete ${stored.relativePath} from the project? The file is removed and cannot be brought back from here.`} + onConfirm={() => void panel.deleteTest(stored.relativePath)} /> + </span>; + })} + {state.tests.length === 0 && <p className="px-1.5 py-1 text-xs text-muted-foreground">No tests in this project's tests/ folder yet.</p>} + </ListGroup> + </PopoverContent> + </Popover>; +} + +/** + * The Testing tool: choose an e2e project, launch its app on the mirrored device, then record a + * deterministic test by tapping the mirror. + * + * Recording is not implemented yet — this is the project-and-launch skeleton. The design, including + * how a tap becomes a selector and when the file is written, is in `docs/testing-panel.md`. + */ +export function TestingView({ panel, deviceApps }: { panel: TestingPanel; deviceApps: DeviceAppsStore }) { + const state = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const devices = useSyncExternalStore(deviceApps.subscribe, deviceApps.getSnapshot); + const device = devices.device; + const booted = device?.state === "Booted" || device?.state === "connected"; + const target = state.targets.find(entry => entry.name === state.targetName) ?? state.targets[0]; + const mismatch = panel.targetMismatch; + const recordable = isRecordableDevice(device) && !mismatch; + const deviceId = device?.udid ?? ""; + const platform = device ? recordablePlatform(device) : "ios"; + // Why the recorder cannot start yet, in the order the user has to fix them. + const blocker = !state.projectPath ? "Choose a project to record against." + : state.loadingProject ? "" + : state.targets.length === 0 ? "No targets found in this project's e2e config." + : !target?.bundleId ? `Target "${target?.name}" declares no bundleId, so there is no app to launch.` + : !booted ? "Boot an iOS simulator or Android emulator to record against it." + : !isRecordableDevice(device) ? "Recording works on an iOS simulator or an Android emulator, not a physical device." + : mismatch; + return <section className="flex h-full min-h-0 flex-col gap-4 overflow-y-auto p-4 [&_button:not(:disabled)]:cursor-pointer" aria-label="Testing"> + <header className="flex flex-wrap items-center gap-2 border-b pb-3"> + <TestingProjectPicker panel={panel} /> + {state.targets.length > 1 && <Select items={state.targets.map(entry => ({ value: entry.name, label: entry.name }))} value={state.targetName} onValueChange={value => { if (value) panel.selectTarget(value); }}> + <SelectTrigger id="testing-target-select" size="sm" className="shrink-0 bg-background hover:bg-muted aria-expanded:bg-muted dark:bg-input/30 dark:hover:bg-input/50" aria-label="Target"><SelectValue>{state.targetName || "Target"}</SelectValue></SelectTrigger> + <SelectContent align="start" alignItemWithTrigger={false} className="w-max min-w-(--anchor-width) max-w-(--available-width)"><SelectGroup> + {state.targets.map(entry => <SelectItem key={entry.name} value={entry.name}>{entry.name}{entry.platform && entry.platform !== entry.name ? ` · ${entry.platform}` : ""}</SelectItem>)} + </SelectGroup></SelectContent> + </Select>} + <TestingDraftPicker panel={panel} /> + {/* Before the controls, so the `ml-auto` below has one thing to push away from and the + status does not end up marooned past the right edge of the group. */} + {state.loadingProject && <span className="flex items-center gap-1 text-xs text-muted-foreground" role="status"><LoaderCircleIcon className="animate-spin" />Reading config…</span>} + <TestingControls panel={panel} recordable={recordable} deviceId={deviceId} platform={platform} /> + </header> + + {state.error && <Alert variant="destructive"> + <AlertTitle>Testing</AlertTitle> + <AlertDescription> + <span className="min-w-0 break-words">{state.error}</span> + <Button type="button" variant="ghost" size="sm" className="self-start" onClick={() => panel.dismissError()}>Dismiss</Button> + </AlertDescription> + </Alert>} + + {blocker && !state.agent?.active && <p className="rounded-md bg-muted/50 px-3 py-2 text-xs text-muted-foreground">{blocker}</p>} + + <AgentRecordingBanner panel={panel} /> + + <TestingRecorder panel={panel} recordable={recordable} deviceId={deviceId} platform={platform} /> + </section>; +} + +/** + * What the panel says while the user's own coding agent is writing the flow. + * + * The recording lives in the server and the panel polls it, so this is the one thing on screen + * that says who is driving — and the Stop button is the reason it has to be: the user asked for + * the recording and must be able to take the mirror back without going via their agent. + * + * Once the agent stops, the banner says what was recorded and that nothing has been saved. From + * there it is an ordinary unsaved recording: the same step list, the same preview, the same Save. + */ +function AgentRecordingBanner({ panel }: { panel: TestingPanel }) { + const state = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const agent = state.agent; + if (!agent || agent.status === "idle") return null; + if (agent.active) return <div data-testing-agent-banner className="flex flex-wrap items-center gap-2 rounded-md border border-primary/40 bg-primary/5 px-3 py-2 text-xs" role="status"> + <BotIcon className="size-4 shrink-0 text-primary" aria-hidden /> + <span className="min-w-0 flex-1 break-words"><span className="font-medium">Agent is recording</span>{agent.goal ? ` — ${agent.goal}` : ""}</span> + <span className="shrink-0 tabular-nums text-muted-foreground">{agent.steps} step{agent.steps === 1 ? "" : "s"}</span> + <Button type="button" size="sm" variant="outline" data-testing-agent-stop className="shrink-0" + title="Stop the agent's recording. The steps it has written stay here for you to adjust." + onClick={() => void panel.stopAgent()}><SquareIcon />Stop</Button> + </div>; + // The handover. It says what is there and what has *not* happened, because "recorded" reads as + // "saved" to anyone who did not watch it: nothing is in the project until the user saves it. + return <p data-testing-agent-handover className="rounded-md border bg-muted/40 px-3 py-2 text-xs text-muted-foreground" role="status"> + {agent.status === "aborted" ? "The agent stopped recording" : "The agent finished recording"} + {` ${agent.steps} step${agent.steps === 1 ? "" : "s"}`}{agent.note ? ` — ${agent.note}` : ""}. + {" "}Nothing has been saved or run: adjust the steps below, then press Save. + </p>; +} + +/** + * Launch, Record and Run test: the three things done to the app, as one control group. + * + * They are one row, one gap and one button size because they are read as one sequence — launch the + * app, record against it, run what was recorded. Launch used to sit outside this group and be + * pushed to the far end of the header by `sm:ml-auto`, which put the first step of that sequence + * furthest from the other two and left it to wrap on its own in a narrow panel. It keeps the solid + * variant: it is the one step that has to happen before the rest mean anything, and that is a + * difference worth seeing — the height, padding, gap and icon treatment are the primitive's own + * and are now identical across all three. + */ +function TestingControls({ panel, recordable, deviceId, platform }: { panel: TestingPanel; recordable: boolean; deviceId: string; platform: DevicePlatform }) { + const state = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + const panelState = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const recording = state.recordingDeviceId === deviceId && deviceId !== ""; + // While an agent is recording, the device is its own: a launch would reset the app under the + // flow, and Record would arm a second driver on one screen. + const agentRecording = panelState.agent?.active === true; + // `ml-auto` on the group, not on any button in it: the header is one flex row, so the free + // space between what names the project and what acts on it belongs to the gap between them. + // The group keeps its own gap and sizes, and on a header too narrow to hold both it wraps whole + // onto the next line rather than crowding or clipping the picker. + return <div data-testing-controls className="flex min-w-0 flex-wrap items-center gap-2 ml-auto"> + <Button type="button" size="sm" data-testing-launch disabled={!panel.canLaunch || agentRecording} onClick={() => void panel.launchApp()}> + {panelState.launching ? <LoaderCircleIcon className="animate-spin" /> : <PlayIcon />} + {panelState.appLaunched ? "Relaunch app" : "Launch app"} + </Button> + {panelState.selectedTestPath ? <Button type="button" size="sm" variant="outline" data-testing-new-from-preview + onClick={() => void panel.newRecording()}><FilePlus2Icon />New recording</Button> : agentRecording + // The agent's own Stop is on the banner, where the goal it is recording for is: one Stop, + // beside what it stops. + ? null + : state.sourceOverride !== undefined + ? <Button type="button" size="sm" variant="outline" data-testing-new-from-source + onClick={() => void panel.newRecording()}><FilePlus2Icon />New recording</Button> + : recording + ? <Button type="button" size="sm" variant="outline" data-testing-stop onClick={() => testRecorder.stop()}><SquareIcon />Stop</Button> + : state.steps.length > 0 + ? <RecordMenu deviceId={deviceId} platform={platform} recordable={recordable} steps={state.steps.length} /> + : <Button type="button" size="sm" variant="outline" data-testing-record disabled={!recordable} + onClick={() => testRecorder.start(deviceId, platform)}><CircleIcon />Record</Button>} + <Button type="button" size="sm" variant="outline" data-testing-run disabled={panelState.selectedTestPath ? !panel.canRunStored : !panel.canRun} + title="Save the test, then run it against this device. Recording stops: the runner needs the device to itself." + onClick={() => void (panelState.selectedTestPath ? panel.runStoredTest() : panel.runTest())}> + {panelState.running ? <LoaderCircleIcon className="animate-spin" /> : <FlaskConicalIcon />}Run test + </Button> + {state.resolving && <span className="flex items-center gap-1 text-xs text-muted-foreground" role="status"><LoaderCircleIcon className="animate-spin" />Reading the screen…</span>} + </div>; +} + +/** + * What Record does when there are already steps. + * + * It used to decide on its own, from whether the recording was dirty: steps survived while they + * were unsaved and were wiped the moment they had been saved — and saving stops the recording, so + * the very next Record threw away the flow the user had just written. The condition was sound and + * invisible, which is the worst a destructive default can be. The question is small enough to ask. + * + * Escape dismisses it without recording: a popover that starts something on the way out is one + * nobody can back out of. + */ +function RecordMenu({ deviceId, platform, recordable, steps }: { deviceId: string; platform: DevicePlatform; recordable: boolean; steps: number }) { + const [open, setOpen] = useState(false); + const choose = (keepSteps: boolean) => { setOpen(false); testRecorder.start(deviceId, platform, keepSteps); }; + const count = `${steps} step${steps === 1 ? "" : "s"}`; + return <Popover open={open} onOpenChange={setOpen}> + <PopoverTrigger asChild> + <Button type="button" size="sm" variant="outline" data-testing-record disabled={!recordable}><CircleIcon />Record</Button> + </PopoverTrigger> + <PopoverContent align="start" className="flex w-64 flex-col gap-1 p-1.5 [&_button:not(:disabled)]:cursor-pointer" + role="menu" aria-label="Record"> + <p className="px-2.5 pb-0.5 pt-1.5 text-[10px] font-medium uppercase tracking-wide text-muted-foreground"> + This flow has {count} + </p> + <MenuRow icon={<PlayIcon className="size-4 shrink-0 text-muted-foreground" />} attributes={{ "data-testing-record-resume": "" }} + title={`Keep the ${count} and record the next one after them`} onClick={() => choose(true)}>Resume</MenuRow> + <MenuRow icon={<Trash2Icon className="size-4 shrink-0 text-muted-foreground" />} attributes={{ "data-testing-record-new": "" }} + title={`Clear the ${count} and start a new flow`} onClick={() => choose(false)}>New</MenuRow> + </PopoverContent> + </Popover>; +} + +/** + * The emitted test, with the lines a step just added called out. + * + * The whole file is re-emitted on every change, so what is new has to be recovered by comparing + * with what was shown last. That is held in a ref rather than state: it is derived from the + * source and recomputing it during the render that shows the source keeps the two in step, where + * an effect would flash the new lines unmarked for a frame. `generation` remounts the marked + * lines so their animation runs again for the next step, instead of only on the first. + */ +function TestPreview({ source, run, highlightedLines, playUntil, onPlayUntil }: { + source: string; run?: TestRunView; highlightedLines?: { from: number; to: number }; + /** Which step each line belongs to, so a line can be run up to. Empty when nothing can be run. */ + playUntil?: ReadonlyMap<number, { stepId: string; label: string }>; + onPlayUntil?: (stepId: string) => void; +}) { + const shown = useRef({ source: "", range: [0, 0] as [number, number], generation: 0 }); + if (shown.current.source !== source) { + const previous = shown.current.source; + shown.current = { + source, + // Nothing was on screen before, so nothing arrived: opening the panel is not a change. + range: previous ? changedLines(previous.split("\n"), source.split("\n")) : [0, 0], + generation: shown.current.generation + 1, + }; + } + const { range: [from, to], generation } = shown.current; + // The preview is a well the panel contains, not a card sitting on it. The muted surface reads + // as recessed in light mode, but in dark mode it is *lighter* than the panel behind it, so the + // well is cut deeper there instead of being lifted out of the pane. + return <pre data-testing-preview className="min-h-0 flex-1 overflow-auto rounded-md border bg-muted/40 p-2 font-mono text-[11px] leading-relaxed dark:bg-black/15"><code> + {source.split("\n").map((line, index) => { + const added = index >= from && index < to; + // The report's lines are 1-based, as the file's own line numbers are. + const mark = run?.lines[index + 1]; + const highlighted = highlightedLines && index + 1 >= highlightedLines.from && index + 1 <= highlightedLines.to; + // Play until here. Only on a line the runner executes: the imports, the comments and the + // closing brace are not somewhere a run can stop, so they are not offered as one. + const play = onPlayUntil ? playUntil?.get(index + 1) : undefined; + return <span key={added ? `new-${generation}-${index}` : `line-${index}`} data-testing-preview-line={index + 1} + className={`testing-preview-line${added ? " is-new" : ""}${highlighted ? " is-step-highlighted" : ""}`}> + {/* Two columns: a fixed gutter and the code. That is why the code does not move when a run + starts, and why a mark stays level with the first row of a line that wraps — the mark is + a sibling of the whole line, not a character at the front of it. + The run's own state is announced once, above the preview; a mark per line would read the + whole file out. So the gutter is decoration, and `aria-hidden` throughout. */} + {/* Only a tick fades. A failing mark is what says where the run broke, so it stays. */} + {/* The file's own line numbers. They are what the runner's report, an error and this panel + all point at, so reading one off the preview should not mean counting rows. Decoration + like the mark beside them — announced nowhere — and outside the selection, so copying + the preview gives the code and not a column of numbers with it. */} + <span className="testing-preview-number" aria-hidden>{index + 1}</span> + <span className={`testing-run-mark${mark ? ` is-${mark}` : ""}${run?.fading && mark === "passed" ? " is-fading" : ""}`} aria-hidden> + {mark === "running" ? <LoaderCircleIcon className="size-3 animate-spin" /> + : mark === "passed" ? <CheckIcon className="size-3" /> + : mark === "failed" ? <CircleXIcon className="size-3" /> : null} + </span> + {/* The gutter holds one thing at a time: hovering a line swaps its mark for the play that + runs the flow up to it, and the mark comes back when the pointer leaves. It is a real + button rather than a click target on the line, so it is reachable by keyboard and says + what it does — the mark beside it is decoration, this is not. */} + {play && <button type="button" data-testing-play-until={index + 1} className="testing-play-until" + title={`Run the test up to and including ${play.label}, and no further`} + aria-label={`Run the test up to ${play.label}, line ${index + 1}`} + onClick={() => onPlayUntil?.(play.stepId)}><PlayIcon className="size-3" /></button>} + <span className="testing-preview-code"> + {highlightCode(line).map((token, position) => + token.kind === "plain" + ? <span key={position}>{token.text}</span> + : <span key={position} className={CODE_TOKEN_CLASS[token.kind]}>{token.text}</span>)} + </span> + </span>; + })} + </code></pre>; +} + +/** A code edit is local until Save; Cancel leaves the current test untouched. */ +function PreviewEditor({ source, onCancel, onSave, saving }: { + source: string; onCancel: () => void; onSave: (source: string) => void; saving: boolean; +}) { + const [value, setValue] = useState(source); + const highlighted = useRef<HTMLPreElement>(null); + const inputRef = useRef<HTMLTextAreaElement>(null); + const [cursor, setCursor] = useState(0); + const [completionOpen, setCompletionOpen] = useState(false); + const [selectedSuggestion, setSelectedSuggestion] = useState(0); + const [completionPosition, setCompletionPosition] = useState({ left: 8, top: 8 }); + const completion = completionOpen ? testSourceSuggestions(value, cursor) : undefined; + const placeCompletion = (input: HTMLTextAreaElement, nextValue: string, position: number) => { + const before = nextValue.slice(0, position); + const lines = before.split("\n"); + const style = getComputedStyle(input); + const fontSize = parseFloat(style.fontSize) || 11; + const lineHeight = parseFloat(style.lineHeight) || fontSize * 1.625; + const characterWidth = fontSize * 0.6; + const left = (parseFloat(style.paddingLeft) || 8) + (lines.at(-1)?.length ?? 0) * characterWidth - input.scrollLeft; + const below = (parseFloat(style.paddingTop) || 8) + lines.length * lineHeight - input.scrollTop; + setCompletionPosition({ + left: Math.max(6, Math.min(left, Math.max(6, input.clientWidth - 220))), + top: Math.max(6, below + 174 > input.clientHeight ? below - 174 : below), + }); + }; + const acceptCompletion = (item: NonNullable<typeof completion>["items"][number]) => { + if (!completion || !inputRef.current) return; + const next = value.slice(0, completion.from) + item.insert + value.slice(completion.to); + const nextCursor = completion.from + item.insert.length - (item.cursorBack ?? 0); + setValue(next); + setCursor(nextCursor); + setSelectedSuggestion(0); + setCompletionOpen(Boolean(testSourceSuggestions(next, nextCursor))); + placeCompletion(inputRef.current, next, nextCursor); + inputRef.current.focus(); + requestAnimationFrame(() => inputRef.current?.setSelectionRange(nextCursor, nextCursor)); + }; + return <div className="flex min-h-0 flex-1 flex-col gap-2" data-testing-source-editor> + <div className="testing-source-editor-code bg-muted/40 dark:bg-black/15"> + <pre ref={highlighted} aria-hidden="true" data-testing-source-highlight><code> + {value.split("\n").map((line, index) => <span key={index} className="testing-source-editor-line"> + <span className="testing-source-editor-number" data-testing-source-line-number>{index + 1}</span> + {highlightCode(line).map((token, position) => <span key={position} className={CODE_TOKEN_CLASS[token.kind]}>{token.text}</span>)} + {line.length === 0 ? "\u200b" : null} + </span>)} + </code></pre> + <textarea ref={inputRef} autoFocus aria-label="Test source" data-testing-source-input spellCheck={false} wrap="off" + aria-autocomplete="list" aria-controls={completion ? "testing-source-suggestions" : undefined} + aria-activedescendant={completion ? `testing-source-suggestion-${selectedSuggestion}` : undefined} + className="testing-source-editor-input" value={value} onChange={event => { + const input = event.currentTarget; + setValue(input.value); + setCursor(input.selectionStart); + setSelectedSuggestion(0); + setCompletionOpen(input.selectionStart === input.selectionEnd); + placeCompletion(input, input.value, input.selectionStart); + }} + onBlur={() => setCompletionOpen(false)} onClick={() => setCompletionOpen(false)} + onScroll={event => { + if (!highlighted.current) return; + highlighted.current.scrollTop = event.currentTarget.scrollTop; + highlighted.current.scrollLeft = event.currentTarget.scrollLeft; + if (completionOpen) placeCompletion(event.currentTarget, value, cursor); + }} + onKeyDown={event => { + if (completion && (event.key === "ArrowDown" || event.key === "ArrowUp")) { + event.preventDefault(); + setSelectedSuggestion(current => (current + (event.key === "ArrowDown" ? 1 : -1) + completion.items.length) % completion.items.length); + return; + } + if (completion && (event.key === "Enter" || event.key === "Tab")) { + event.preventDefault(); + acceptCompletion(completion.items[selectedSuggestion]); + return; + } + if (event.key === "Escape" && completion) { event.preventDefault(); setCompletionOpen(false); return; } + if (["ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown"].includes(event.key)) setCompletionOpen(false); + if (event.key !== "Tab" || event.shiftKey) return; + event.preventDefault(); + const input = event.currentTarget; + const start = input.selectionStart, end = input.selectionEnd; + const next = `${value.slice(0, start)} ${value.slice(end)}`; + setValue(next); + requestAnimationFrame(() => { input.selectionStart = input.selectionEnd = start + 2; }); + }} /> + {completion && <div id="testing-source-suggestions" role="listbox" aria-label="Test code suggestions" + data-testing-source-suggestions className="testing-source-suggestions" style={completionPosition}> + {completion.items.map((item, index) => <button key={item.label} type="button" role="option" + id={`testing-source-suggestion-${index}`} aria-selected={index === selectedSuggestion} + data-testing-source-suggestion={item.label} className="testing-source-suggestion" + onMouseDown={event => event.preventDefault()} onMouseEnter={() => setSelectedSuggestion(index)} + onClick={() => acceptCompletion(item)}>{item.label}{item.detail + ? <span className="block truncate text-[10px] text-muted-foreground">{item.detail}</span> + : <span className="text-muted-foreground">{item.insert.startsWith(item.label) ? item.insert.slice(item.label.length) : ""}</span>}</button>)} + </div>} + </div> + <div className="flex justify-end gap-2"> + <Button type="button" size="sm" variant="ghost" data-testing-source-cancel onClick={onCancel}>Cancel</Button> + <Button type="button" size="sm" data-testing-source-save disabled={saving || !value.trim()} + onClick={() => onSave(value)}><SaveIcon />Save</Button> + </div> + </div>; +} + +/** Source-backed drafts and project files have the same editable awaited-call rows. */ +function SourceStepList({ source, mode, onEdit, onHoverLine }: { + source: string; mode: "source" | "stored"; onEdit: (source: string) => void; + onHoverLine?: (line: number | undefined) => void; +}) { + const steps = storedTestSteps(source); + const [draggingLine, setDraggingLine] = useState<number>(); + const [overLine, setOverLine] = useState<number>(); + const prefix = mode === "stored" ? "data-testing-stored" : "data-testing-source"; + return <ol className="flex flex-col gap-1 text-xs" {...{ [`${prefix}-steps`]: "" }}> + {steps.map((step, index) => { + const editable = Boolean(storedTestUntilSource(source, step.line)); + return <li key={step.line}> + <div {...{ [`${prefix}-step`]: String(step.line) }} draggable={editable} + className={`group/step flex items-center gap-2 rounded-md border px-2 py-1.5 text-xs ${draggingLine === step.line ? "opacity-50" : ""} ${overLine === step.line && draggingLine !== step.line ? "border-primary" : ""}`} + onMouseEnter={() => onHoverLine?.(step.line)} onMouseLeave={() => onHoverLine?.(undefined)} + onDragStart={event => { if (!editable) return; setDraggingLine(step.line); event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData("text/plain", String(step.line)); }} + onDragEnd={() => { setDraggingLine(undefined); setOverLine(undefined); }} + onDragOver={event => { if (draggingLine === undefined || !editable) return; event.preventDefault(); event.dataTransfer.dropEffect = "move"; setOverLine(step.line); }} + onDrop={event => { + event.preventDefault(); + if (draggingLine !== undefined && editable) onEdit(moveStoredStep(source, draggingLine, step.line)); + setDraggingLine(undefined); setOverLine(undefined); + }}> + <GripVerticalIcon aria-hidden className={`size-3.5 shrink-0 text-muted-foreground opacity-40 group-hover/step:opacity-100 ${editable ? "cursor-grab" : ""}`} /> + <span className="w-4 shrink-0 text-right text-muted-foreground">{index + 1}</span> + <button type="button" className="min-w-0 flex-1 truncate text-left font-mono" title={step.label} + onFocus={() => onHoverLine?.(step.line)} onBlur={() => onHoverLine?.(undefined)}>{step.label}</button> + <SourceStepMenu source={source} step={step} index={index} mode={mode} onEdit={onEdit} editable={editable} /> + <Button type="button" variant="ghost" size="icon-xs" aria-label={`Remove step ${index + 1}`} + {...{ [`${prefix}-remove`]: String(step.line) }} disabled={!editable} + title={editable ? "Remove this step" : "Edit this multi-line call in Preview"} + onClick={() => onEdit(removeStoredStep(source, step.line))}><XIcon /></Button> + </div> + </li>; + })} + {steps.length === 0 && <li className="rounded-lg border border-dashed px-3 py-5 text-center text-muted-foreground">No awaited calls found in this file.</li>} + </ol>; +} + +function SourceStepMenu({ source, step, index, mode, onEdit, editable }: { + source: string; step: ReturnType<typeof storedTestSteps>[number]; index: number; + mode: "source" | "stored"; onEdit: (source: string) => void; editable: boolean; +}) { + const [open, setOpen] = useState(false); + const leaving = useRef<ReturnType<typeof setTimeout>>(undefined); + const prefix = mode === "stored" ? "data-testing-stored" : "data-testing-source"; + const commands = editable ? storedStepCommands(step) : []; + const hold = () => { if (leaving.current) clearTimeout(leaving.current); }; + const close = () => { hold(); setOpen(false); }; + const release = () => { + hold(); + const closeWhenBranchLeaves = () => { + if (document.querySelector(`[${prefix}-commands-panel="${step.line}"]`)) + leaving.current = setTimeout(closeWhenBranchLeaves, 120); + else close(); + }; + leaving.current = setTimeout(closeWhenBranchLeaves, 120); + }; + useEffect(() => () => hold(), []); + return <Popover open={open} onOpenChange={setOpen}> + <PopoverTrigger asChild><Button type="button" variant="ghost" size="icon-xs" + {...{ [`${prefix}-step-menu`]: String(step.line) }} aria-label={`Edit step ${index + 1}`} + className="shrink-0" + onMouseEnter={() => setOpen(true)}><MoreHorizontalIcon /></Button></PopoverTrigger> + <PopoverContent align="end" className="flex w-60 max-h-(--radix-popover-content-available-height) flex-col gap-1 overflow-y-auto p-1.5 [&_button:not(:disabled)]:cursor-pointer" + role="menu" aria-label={`Step ${index + 1}`} onMouseEnter={hold} onMouseLeave={release}> + <MenuRow icon={<CopyIcon className="size-4 shrink-0 text-muted-foreground" />} title={editable ? "Duplicate this step" : "Edit this multi-line call in Preview"} + attributes={{ [`${prefix}-duplicate`]: String(step.line) }} disabled={!editable} + onClick={() => { close(); onEdit(duplicateStoredStep(source, step.line)); }}>Duplicate</MenuRow> + <Branch label={`Change command for step ${index + 1}`} width={STEP_COMMANDS_WIDTH} align="top" side="left" + attributes={{ [`${prefix}-commands-panel`]: String(step.line) }} + renderRow={handlers => <MenuRow chevron icon={<OfferIcon kind="tap" />} + attributes={{ [`${prefix}-commands`]: String(step.line) }} + onOpen={handlers.onOpen} onClick={handlers.onClick}>Change command</MenuRow>}> + {commands.length ? commands.map(command => <MenuRow key={command.id} title={`Change to ${command.label}`} + attributes={{ [`${prefix}-command`]: command.id }} + onClick={() => { close(); onEdit(changeStoredStepCommand(source, step.line, command.id)); }}>{command.label}</MenuRow>) + : <p className="px-2 py-1 text-xs text-muted-foreground">No compatible commands for this call. Edit it in Preview.</p>} + </Branch> + </PopoverContent> + </Popover>; +} + +/** A project file's awaited calls can be edited in a temporary working copy. */ +function StoredTestPreview({ panel, source, path, loading }: { panel: TestingPanel; source: string; path: string; loading: boolean }) { + const panelState = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const recorder = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + const [highlightedLine, setHighlightedLine] = useState<number>(); + const [editing, setEditing] = useState(false); + const steps = storedTestSteps(source); + const playUntil = new Map<number, { stepId: string; label: string }>(); + if (recorder.run?.state !== "running") { + for (const [index, line] of source.split("\n").entries()) { + if (/^\s*await\s+.+;\s*$/.test(line)) playUntil.set(index + 1, { stepId: String(index + 1), label: `line ${index + 1}` }); + } + } + return <div className="flex min-h-0 flex-1 flex-col gap-2" data-testing-stored-preview> + <div className="flex flex-wrap items-center gap-2 text-xs"> + <span className="min-w-0 flex-1 truncate text-muted-foreground">{path}</span> + {panelState.selectedTestStaged && <> + <span className="text-amber-600 dark:text-amber-400" role="status">Unsaved edits are temporary and may be lost until you Save.</span> + <Button type="button" size="sm" variant="ghost" data-testing-stored-undo disabled={panelState.saving} + onClick={() => void panel.undoStoredTest()}>Undo edits</Button> + <Button type="button" size="sm" data-testing-stored-save disabled={panelState.saving} + onClick={() => void panel.saveStoredTest()}><SaveIcon />Save</Button> + </>} + </div> + {loading ? <p className="flex items-center gap-1 text-xs text-muted-foreground" role="status"><LoaderCircleIcon className="size-3 animate-spin" />Reading test…</p> + : <ResizablePanelGroup id="testing-stored-panels" orientation="horizontal" className="min-h-0 flex-1 gap-3" aria-label="Saved test preview and steps"> + <ResizablePanel id="testing-stored-source" defaultSize="60%" minSize="20%" maxSize="85%"> + <section className="flex h-full min-h-0 min-w-0 flex-col gap-1.5 overflow-hidden rounded-lg border p-3" aria-label="Preview"> + <p className="flex items-center gap-2 text-xs font-medium">Preview + <Button type="button" size="sm" variant="ghost" className={editing ? "bg-accent text-accent-foreground" : undefined} data-testing-source-edit aria-pressed={editing} + disabled={editing || panelState.running || panelState.saving} onClick={() => setEditing(true)}>Edit</Button> + {recorder.run?.state === "running" && <span role="status" className="text-muted-foreground">Running…</span>} + {recorder.run?.state === "passed" && <span role="status" className="text-emerald-600">Passed</span>} + {recorder.run?.state === "failed" && <span role="status" className="text-red-600">{recorder.run.error ?? "Failed"}</span>} + </p> + {editing ? <PreviewEditor source={source} saving={panelState.saving} + onCancel={() => setEditing(false)} onSave={value => { panel.editStoredTest(value); setEditing(false); void panel.saveStoredTest(); }} /> + : <TestPreview source={source} run={recorder.run} + highlightedLines={highlightedLine === undefined ? undefined : { from: highlightedLine, to: highlightedLine }} + playUntil={playUntil} onPlayUntil={panel.canRunStored ? line => void panel.runStoredTest(Number(line)) : undefined} />} + </section> + </ResizablePanel> + <ResizableHandle withHandle aria-label="Resize the preview and the steps" /> + <ResizablePanel id="testing-stored-steps" defaultSize="40%" minSize="15%" maxSize="80%"> + <section className="flex h-full min-h-0 min-w-0 flex-col gap-2 overflow-hidden rounded-lg border p-3" aria-label="Steps"> + <p className="flex items-center gap-2 text-xs font-medium">Steps <span className="font-normal text-muted-foreground">{steps.length}</span></p> + <div className="min-h-0 flex-1 overflow-y-auto pr-1"><SourceStepList source={source} mode="stored" + onEdit={value => panel.editStoredTest(value)} onHoverLine={setHighlightedLine} /></div> + </section> + </ResizablePanel> + </ResizablePanelGroup>} + </div>; +} + +/** + * The recorder: the steps taken so far and the test they emit. The taps themselves are captured on + * the mirror canvas by the simulator panel, which holds the device connection; this is the readout + * and the controls. + */ +function TestingRecorder({ panel, recordable, deviceId }: { panel: TestingPanel; recordable: boolean; deviceId: string; platform: DevicePlatform }) { + const state = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + const panelState = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const recording = state.recordingDeviceId === deviceId && deviceId !== ""; + // Not while the agent is still writing the flow: saving half of one is a file the user did not + // ask for, and the agent is told to hand over rather than to save. + const saveable = Boolean(panelState.projectPath) && (state.steps.length > 0 || Boolean(state.sourceOverride)) && !panelState.saving && !panelState.agent?.active; + const [editing, setEditing] = useState(false); + const sourceSteps = state.sourceOverride === undefined ? undefined : storedTestSteps(state.sourceOverride); + const [hoveredStep, setHoveredStep] = useState<string>(); + const [focusedStep, setFocusedStep] = useState<string>(); + /** + * The highlight goes the moment the pointer leaves the row, and a row that has already been left + * cannot take it from the one the pointer is on now. + * + * What made leaving a step *re-light* its lines was never the clearing — it was the preview's + * own arrival animation being cancelled while the line was highlighted and restarted when it no + * longer was; see `.is-step-highlighted` in `style.css`. With that gone, clearing on the row's + * own leave is exactly right. The guard stays because the browser dispatches the leave of one + * row and the enter of the next together, and the order is not guaranteed to be the one the eye + * follows: clearing only when the id still matches means the row under the pointer wins + * whichever event lands last. + */ + const hoverStep = useCallback((stepId: string | undefined) => setHoveredStep(stepId), []); + const unhoverStep = useCallback((stepId: string) => + setHoveredStep(current => current === stepId ? undefined : current), []); + const highlightedStep = hoveredStep ?? focusedStep; + const spans = testRecorder.emitted.spans; + const highlightedLines = spans.find(span => span.stepId === highlightedStep); + /** + * Which step each line of the preview came from, so a line can offer to run the flow up to it. + * + * Only the awaited calls, which are the lines the runner reports a step for. A step can carry a + * TODO comment above its call, and a comment is not somewhere a run can stop: a play beside one + * would promise a line the runner never executes. The gutter marks exactly these lines too, so + * what offers to be run and what comes back marked are the same set. + * + * Nothing is built while a run is in flight — the gutter is the run's own then, and a second run + * cannot start anyway. + * + * Built on every render, like the spans it reads: both are one pass over the steps, and the + * file is re-emitted whenever those change, so there is nothing a memo could hold on to. + */ + const playUntil = new Map<number, { stepId: string; label: string }>(); + if (state.run?.state !== "running" && sourceSteps === undefined) { + // The app opening is the first thing the runner does and the first thing a flow breaks on, so + // it is somewhere a run can stop too — the one line of the header that is a call at all. + const open = testRecorder.emitted.calls[0]; + if (open !== undefined) playUntil.set(open, { stepId: OPEN_STEP, label: "the app opening" }); + for (const [index, span] of spans.entries()) { + const step = state.steps[index]; + const label = step ? `step ${index + 1} (${step.label})` : `step ${index + 1}`; + for (const line of span.calls) playUntil.set(line, { stepId: span.stepId, label }); + } + } + const playUntilStep = useCallback((stepId: string) => { void panel.runTest({ untilStepId: stepId }); }, [panel]); + if (panelState.selectedTestPath) return <StoredTestPreview key={panelState.selectedTestPath} panel={panel} source={panelState.selectedTestSource} + path={panelState.selectedTestPath} loading={panelState.loadingTest} />; + return <div className="flex min-h-0 flex-1 flex-col gap-3"> + {!recordable && !recording && !panelState.agent?.active && <p className="text-xs text-muted-foreground">Recording needs a booted iOS simulator or Android emulator in the mirror.</p>} + {panelState.agent?.active && <p className="text-xs text-muted-foreground"> + The mirror is the agent's while it records, so clicking it does nothing. Press Stop to take it back. + </p>} + {recording && !panelState.agent?.active && <p className="text-xs text-muted-foreground"> + Click the mirror to drive the app; nothing is recorded. Right-click an element for the steps it + supports, and scroll over the outline to pick the element around it. + </p>} + + {state.error && <Alert variant="destructive"> + <AlertTitle>Recorder</AlertTitle> + <AlertDescription> + <span className="min-w-0 break-words">{state.error}</span> + <Button type="button" variant="ghost" size="sm" className="self-start" onClick={() => testRecorder.dismissError()}>Dismiss</Button> + </AlertDescription> + </Alert>} + + {/* The preview and the step list are dragged apart rather than fixed at 3fr/2fr: how much of + the pane each deserves depends on the flow being written — a long file to read, or a long + list to reorder — and that changes while one test is being recorded. The defaults are the + panel's own share of the workspace: with the mirror at 25%, these are the 45% and 30% the + preview and the steps get. + A row at every width, with no stacked form. Stacking gave each pane 60% and 40% of the + *height* left over under the mirror, which is how the three panes ended up at 60/22/16 + instead of 25/45/30; the panes have their own minimum widths, and the workspace keeps the + mirror beside them wherever there is width for a row. */} + <ResizablePanelGroup id="testing-workspace-panels" orientation="horizontal" + data-testing-recorder-workspace className="min-h-0 flex-1 gap-3" + aria-label="Preview and steps"> + <ResizablePanel id="testing-preview-resizable" defaultSize="60%" minSize="20%" maxSize="85%"> + <section data-testing-preview-pane className="flex h-full min-h-0 min-w-0 flex-col gap-1.5 overflow-hidden rounded-lg border p-3"> + <p className="flex flex-wrap items-center gap-2 text-xs font-medium"> + <span>Preview</span> + {!panelState.agent?.active && <Button type="button" size="sm" variant="ghost" className={editing ? "bg-accent text-accent-foreground" : undefined} data-testing-source-edit aria-pressed={editing} + disabled={editing || panelState.running || panelState.saving} onClick={() => setEditing(true)}>Edit</Button>} + {state.run?.state === "running" && <span className="flex items-center gap-1 font-normal text-muted-foreground" role="status"> + <LoaderCircleIcon className="size-3 animate-spin" />Running + </span>} + {state.run?.state === "passed" && <span className="flex items-center gap-1 font-normal text-emerald-600 dark:text-emerald-400" role="status"><CheckIcon className="size-3" />Passed</span>} + {state.run?.state === "failed" && <span className="flex min-w-0 items-center gap-1 break-words font-normal text-red-600 dark:text-red-400" role="status"> + <CircleXIcon className="size-3 shrink-0" />{state.run.error ?? "Failed"} + </span>} + {/* The runner's own number, beside its verdict and on a failure too: how long a run took + before it gave up is part of reading it. It is not the panel's `ui.testing.run` + timing, which includes spawning the CLI. */} + {state.run && state.run.state !== "running" && state.run.durationMs !== undefined + && <span data-testing-run-duration className="font-normal tabular-nums text-muted-foreground" + title="How long the runner reported the run taking.">{formatRunDuration(state.run.durationMs)}</span>} + {/* A verdict about part of the flow must not read as a verdict about the test: the run + was asked to stop at a step, so what it passed is everything up to that line. */} + {state.run?.untilLine !== undefined && <span data-testing-run-until className="font-normal text-muted-foreground" + title="This run stops at the step you played to; the steps below it are not run.">· to line {state.run.untilLine}</span>} + {/* The verdict is still about a run that happened; the lines it marked have moved. */} + {state.run?.stale && <span data-testing-run-stale className="font-normal text-muted-foreground" + title="The steps changed after this run, so the line marks no longer point at the same steps.">· before your edits</span>} + </p> + {state.run?.output && <pre className="max-h-32 overflow-auto rounded-md border bg-muted/40 p-2 font-mono text-[10px] whitespace-pre-wrap">{state.run.output}</pre>} + {editing ? <PreviewEditor source={testRecorder.source} saving={panelState.saving} + onCancel={() => setEditing(false)} onSave={value => { testRecorder.editSource(value); setEditing(false); void panel.saveTest(); }} /> + : <TestPreview source={testRecorder.source} run={state.run} highlightedLines={sourceSteps ? undefined : highlightedLines} + playUntil={playUntil} onPlayUntil={panel.canRun && !sourceSteps ? playUntilStep : undefined} />} + </section> + </ResizablePanel> + + <ResizableHandle withHandle aria-label="Resize the preview and the steps" /> + + <ResizablePanel id="testing-steps-resizable" defaultSize="40%" minSize="15%" maxSize="80%"> + <div data-testing-editor-pane className="flex h-full min-h-0 min-w-0 flex-col gap-3 overflow-hidden rounded-lg border p-3"> + <TestDescription panel={panel} saveable={saveable} /> + + <div className="flex min-h-0 flex-1 flex-col gap-2 border-t pt-3"> + <div className="flex items-center gap-2"> + <h2 className="text-xs font-medium">Steps</h2> + {(sourceSteps ?? state.steps).length > 0 && <span className="text-xs text-muted-foreground">{(sourceSteps ?? state.steps).length}</span>} + <ClearStepsButton steps={(sourceSteps ?? state.steps).length} /> + </div> + <div className="min-h-0 flex-1 overflow-y-auto pr-1">{sourceSteps + ? <SourceStepList source={testRecorder.source} mode="source" onEdit={value => testRecorder.editSource(value)} /> + : <StepList steps={state.steps} onHoverStep={hoverStep} onUnhoverStep={unhoverStep} onFocusStep={setFocusedStep} />}</div> + </div> + + {panelState.overwritePath && <Alert> + <AlertTitle>Replace the existing file?</AlertTitle> + <AlertDescription> + <span className="min-w-0 break-words">{panelState.overwritePath} already exists.</span> + <span className="flex gap-2"> + <Button type="button" size="sm" data-testing-overwrite onClick={() => void panel.saveTest(true)}>Replace</Button> + <Button type="button" size="sm" variant="ghost" onClick={() => panel.cancelOverwrite()}>Cancel</Button> + </span> + </AlertDescription> + </Alert>} + + {panelState.savedPath && !panel.dirty && <p className="text-xs text-muted-foreground" role="status">Saved to {panelState.savedPath}</p>} + {panelState.savedPath && panel.dirty && <p className="text-xs text-amber-600" role="status">Unsaved steps since {panelState.savedPath} was written.</p>} + </div> + </ResizablePanel> + </ResizablePanelGroup> + </div>; +} + +/** + * What the test is called and where it is written, as one collapsible section. + * + * The two fields and Save used to sit permanently above the step list, taking a third of a pane + * that is mostly used for the steps: they are touched once per recording — at the end — while the + * list beside them is touched on every tap. Folded away they are one row, and the row says what is + * inside it, so nothing is hidden from someone who has not opened it yet. It opens itself when + * there is something to save and nothing has been written yet, which is exactly the moment the + * fields are wanted. + */ +function TestDescription({ panel, saveable }: { panel: TestingPanel; saveable: boolean }) { + const state = useSyncExternalStore(testRecorder.subscribe, testRecorder.getSnapshot); + const panelState = useSyncExternalStore(panel.subscribe, panel.getSnapshot); + const [open, setOpen] = useState(false); + return <section className="flex min-w-0 flex-col" aria-label="Test Description"> + <button type="button" data-testing-description-toggle aria-expanded={open} + className="flex min-w-0 items-center gap-1.5 rounded-md px-1 py-1 text-left text-xs font-medium hover:bg-accent hover:text-accent-foreground" + onClick={() => setOpen(value => !value)}> + {open ? <ChevronDownIcon className="size-3.5 shrink-0 text-muted-foreground" /> : <ChevronRightIcon className="size-3.5 shrink-0 text-muted-foreground" />} + <span className="shrink-0">Test Description</span> + {/* Closed, the row is the readout: the name the flow will be saved under, clipped at a width + of its own so it degrades the same way wherever the pane ends. The file name is *not* + here — at the width this pane actually gets, a name and a path side by side were two + truncated strings reading as neither. The path is one click away, in the File name field + below, which is where it is edited anyway. */} + <span data-testing-description-title className="ml-auto min-w-0 max-w-[24ch] truncate font-normal text-muted-foreground" + title={state.title.trim() || "Untitled flow"}> + {state.title.trim() || "Untitled flow"} + </span> + </button> + {open && <div data-testing-description className="grid gap-3 pt-2 lg:grid-cols-[minmax(0,3fr)_minmax(0,5fr)]"> + <label className="flex min-w-0 flex-col gap-1.5 text-xs font-medium"> + <span>Test title</span> + <input type="text" data-testing-title defaultValue={state.title} onBlur={event => testRecorder.setTitle(event.target.value)} + className="min-w-0 rounded-md border bg-background px-2.5 py-1.5 font-normal" aria-label="Test title" /> + </label> + <label className="flex min-w-0 flex-col gap-1.5 text-xs font-medium"> + <span>File name</span> + <span className="flex min-w-0 items-center gap-2"> + <span className="flex min-w-0 flex-1 items-center overflow-hidden rounded-md border bg-background focus-within:ring-2 focus-within:ring-ring/50"> + <code className="shrink-0 border-r bg-muted/50 px-2.5 py-1.5 font-normal text-muted-foreground">tests/</code> + <input type="text" data-testing-filename defaultValue={panelState.fileName} placeholder={`${testFileSlug(state.title)}.e2e.ts`} + onBlur={event => panel.setFileName(event.target.value)} className="min-w-0 flex-1 bg-transparent px-2.5 py-1.5 font-normal outline-none" aria-label="Test file name" /> + </span> + <Button type="button" size="sm" data-testing-save disabled={!saveable} onClick={() => void panel.saveTest()}> + {panelState.saving ? <LoaderCircleIcon className="animate-spin" /> : <SaveIcon />}Save + </Button> + </span> + </label> + </div>} + </section>; +} + +/** + * Clear, as an icon that asks first. + * + * It throws away every step of the flow on screen, and there is no undo: the steps are the whole + * of what was recorded. It used to be one unguarded click of a button labelled "Clear", sitting at + * the end of the same row as the step count. Now it is the icon alone, tinted like the destructive + * thing it is, and the question names how much is about to go. + */ +function ClearStepsButton({ steps }: { steps: number }) { + const [open, setOpen] = useState(false); + const count = `${steps} step${steps === 1 ? "" : "s"}`; + return <Popover open={open} onOpenChange={setOpen}> + <PopoverTrigger asChild> + <Button type="button" size="icon-xs" variant="ghost" data-testing-clear disabled={steps === 0} + className="ml-auto text-destructive hover:bg-destructive/10 hover:text-destructive" + aria-label="Clear test" title="Clear every step of this test"><Trash2Icon /></Button> + </PopoverTrigger> + <PopoverContent align="end" className="flex w-60 flex-col gap-2 p-2.5" role="alertdialog" aria-label="Clear test"> + <p className="text-xs">Clear all {count}? The recording cannot be brought back.</p> + <div className="flex justify-end gap-2"> + <Button type="button" size="sm" variant="ghost" onClick={() => setOpen(false)}>Cancel</Button> + <Button type="button" size="sm" variant="destructive" data-testing-clear-confirm + onClick={() => { setOpen(false); testRecorder.clearSteps(); }}>Clear</Button> + </div> + </PopoverContent> + </Popover>; +} + +/** + * The recorded steps, in the order the file emits them. + * + * Every row is editable in place: a step can be dragged to another position, copied, or pointed at + * a different command while keeping the element it named. All three go through the recorder, so + * the numbering, the ambiguous badge and the preview are recomputed from one list rather than kept + * in step by hand. + */ +function StepList({ steps, onHoverStep, onUnhoverStep, onFocusStep }: { + steps: readonly RecordedStep[]; + onHoverStep: (stepId: string | undefined) => void; + /** Clears the highlight, but only if this row is still the one holding it. */ + onUnhoverStep: (stepId: string) => void; + onFocusStep: (stepId: string | undefined) => void; +}) { + // The row being dragged and the gap it would land in. Held here rather than in the recorder: + // a drag that is abandoned must leave the recorded flow exactly as it was. + const [dragging, setDragging] = useState<string | undefined>(); + const [over, setOver] = useState<number | undefined>(); + const drop = (index: number) => { + if (dragging) testRecorder.moveStep(dragging, index); + setDragging(undefined); setOver(undefined); + }; + // The rows own the highlight between them; the list adds nothing. An unconditional clear here + // fires for the list *and* every ancestor leave, which is one more way for the highlight to be + // taken from the row the pointer is actually on. + return <ol className="flex flex-col gap-1 text-xs"> + {steps.map((step, index) => <li + key={step.id} + draggable + data-testing-step={step.id} + className={`group/step flex items-center gap-2 rounded-md border px-2 py-1.5 ${dragging === step.id ? "opacity-50" : ""} ${over === index && dragging !== step.id ? "border-primary" : ""}`} + onMouseEnter={() => onHoverStep(step.id)} + onMouseLeave={() => onUnhoverStep(step.id)} + onFocus={() => onFocusStep(step.id)} + onBlur={event => { if (!event.currentTarget.contains(event.relatedTarget as Node | null)) onFocusStep(undefined); }} + onDragStart={event => { setDragging(step.id); event.dataTransfer.effectAllowed = "move"; event.dataTransfer.setData("text/plain", step.id); }} + onDragEnd={() => { setDragging(undefined); setOver(undefined); }} + onDragOver={event => { if (!dragging) return; event.preventDefault(); event.dataTransfer.dropEffect = "move"; setOver(index); }} + onDrop={event => { event.preventDefault(); drop(index); }} + > + <GripVerticalIcon aria-hidden className="size-3.5 shrink-0 cursor-grab text-muted-foreground opacity-40 group-hover/step:opacity-100" /> + <span className="w-4 shrink-0 text-right text-muted-foreground">{index + 1}</span> + <span className="min-w-0 flex-1 truncate" title={step.label}>{step.label}</span> + {/* Both badges are about a locator, and an agent step has none: nothing was resolved, so + nothing can be ambiguous or fall back to a coordinate. */} + {!isAgentStep(step) && step.ambiguous && <span className="shrink-0 text-amber-600" title="This matched more than one element. Add a testID to pin it.">ambiguous</span>} + {!isAgentStep(step) && step.selector.kind === "point" && <span className="shrink-0 text-amber-600" title="Nothing resolved here, so the test taps a coordinate.">coordinates</span>} + {isAgentStep(step) && <span className="shrink-0 text-muted-foreground" title="Resolved by the agent at run time, not by a locator.">agent</span>} + <StepMenu step={step} index={index} /> + <Button type="button" variant="ghost" size="icon-xs" aria-label={`Remove step ${index + 1}`} onClick={() => testRecorder.removeStep(step.id)}><XIcon /></Button> + </li>)} + {steps.length === 0 && <li className="rounded-lg border border-dashed px-3 py-5 text-center text-muted-foreground">No steps yet. Right-click an element in the mirror to add one.</li>} + </ol>; +} + +/** How wide the step's command branch is: a list of short command names. */ +const STEP_COMMANDS_WIDTH = 230; + +/** + * One step's own menu, revealed by pointing at its row. + * + * "Change command" is the mirror's "All commands" list, offered against the element this step + * already named: the commands are derived from what that element carried when it was recorded + * (see `PinnedStep.target`), so a text field still offers `fill` and an element with text still + * offers the text matchers, long after the screen has moved on. + * + * An agent step is offered Duplicate alone. It has no element to re-point — and converting it to a + * pinned command would have to invent a locator, while converting a pinned step the other way + * would throw away the one it was recorded for. Agentic is recorded over the mirror, where there + * is a screen to describe. + */ +function StepMenu({ step, index }: { step: RecordedStep; index: number }) { + const [open, setOpen] = useState(false); + const leaving = useRef<ReturnType<typeof setTimeout>>(undefined); + const offers = isAgentStep(step) ? [] : allStepOffers(step.selector, step.target); + const hold = () => { if (leaving.current) clearTimeout(leaving.current); }; + const close = () => { hold(); setOpen(false); }; + const release = () => { + hold(); + const closeWhenBranchesLeave = () => { + if (document.querySelector(`[data-testing-step-commands-panel="${step.id}"]`)) { + leaving.current = setTimeout(closeWhenBranchesLeave, 120); + } else close(); + }; + leaving.current = setTimeout(closeWhenBranchesLeave, 120); + }; + useEffect(() => () => hold(), []); + return <Popover open={open} onOpenChange={setOpen}> + <PopoverTrigger asChild> + <Button type="button" variant="ghost" size="icon-xs" data-testing-step-menu={step.id} + aria-label={`Edit step ${index + 1}`} + // Revealed by the hover, but never hidden from the keyboard: `focus-visible` keeps it + // reachable by tabbing, where opacity alone would make it a control nobody can find. + className={`shrink-0 opacity-0 transition-opacity group-hover/step:opacity-100 focus-visible:opacity-100 ${open ? "opacity-100" : ""}`} + onMouseEnter={() => setOpen(true)}><MoreHorizontalIcon /></Button> + </PopoverTrigger> + <PopoverContent align="end" className="flex w-60 max-h-(--radix-popover-content-available-height) flex-col gap-1 overflow-y-auto p-1.5 [&_button:not(:disabled)]:cursor-pointer" + role="menu" aria-label={`Step ${index + 1}`} onMouseEnter={hold} onMouseLeave={release}> + <MenuRow icon={<CopyIcon className="size-4 shrink-0 text-muted-foreground" />} attributes={{ "data-testing-step-duplicate": step.id }} + title="Add a copy of this step at the end of the list" + onClick={() => { close(); testRecorder.duplicateStep(step.id); }}>Duplicate</MenuRow> + {/* The command list hangs *beside* this menu rather than replacing it, as every other + cascading menu in the panel does. Replacing it took the step away from under the user and + needed a Back row to undo itself; the branch keeps the row it came from on screen, so the + trail from the step to the command reads in one glance. It opens on the left because the + step list is the rightmost pane in the workspace: to the right is the window's edge, and + to the left is the preview, which is where there is room for it. */} + {!isAgentStep(step) && <Branch + label={`Change command for step ${index + 1}`} width={STEP_COMMANDS_WIDTH} align="top" side="left" + attributes={{ "data-element": "testing-step-commands", "data-testing-step-commands-panel": step.id }} + renderRow={handlers => <MenuRow chevron icon={<OfferIcon kind={step.kind} />} + attributes={{ "data-testing-step-commands": step.id }} + title={`Currently ${OFFER_LABEL[step.kind]}`} + onOpen={handlers.onOpen} onClick={handlers.onClick}>Change command</MenuRow>} + > + <StepCommandList commands={offers} initialTimeoutMs={!isAgentStep(step) ? step.timeoutMs : undefined} + onChoose={(kind, value, timeoutMs) => { close(); testRecorder.changeStepCommand(step.id, kind, value, timeoutMs); }} /> + </Branch>} + </PopoverContent> + </Popover>; +} diff --git a/packages/ui/src/device-workspace.ts b/packages/ui/src/device-workspace.ts index fec15af6..263b3f3f 100644 --- a/packages/ui/src/device-workspace.ts +++ b/packages/ui/src/device-workspace.ts @@ -16,7 +16,8 @@ type Platform = "ios" | "android"; export class DeviceWorkspace { readonly panels: readonly [ReturnType<typeof createSimulatorPanel>, ReturnType<typeof createSimulatorPanel>]; private readonly tools: Pick<WorkspaceTools, "context" | "deviceApps">; - private readonly devices: WorkspaceDevices; + /** Assigned after the panels, which call back into it only once they are listing devices. */ + private devices!: WorkspaceDevices; private readonly connectionFailed: (error: unknown) => void; private activePlatform: Platform = "ios"; @@ -32,6 +33,9 @@ export class DeviceWorkspace { return createSimulatorPanel(app, root, platform, tools.context, (_device, active) => { if (active) this.activePlatform = platform; this.updateSelection(); + // Discovery finding a device is also the answer to a layout that resolved to "none" + // because the first device list came back before the backend had found anything. + this.devices?.settleLayout(); }); }; this.panels = [create("ios"), create("android")]; diff --git a/packages/ui/src/popover-placement.ts b/packages/ui/src/popover-placement.ts new file mode 100644 index 00000000..ee6cb326 --- /dev/null +++ b/packages/ui/src/popover-placement.ts @@ -0,0 +1,102 @@ +/** + * Where a popover goes, in the window's own coordinates. + * + * The Testing panel's element menu used to be positioned inside the stage — the box the mirror + * sits in — which clips what it contains and is often shorter than the menu itself, so a menu on + * a bottom tab was cut off rather than moved. Placement is against the window instead, and the + * caller draws the popover in a portal so nothing between it and the window can clip it. + * + * It is kept apart from the component so the arithmetic can be checked against real numbers — + * a short panel, a narrow one — rather than only through a DOM with no layout in it. + */ + +/** A rectangle in the window's own coordinates, as `getBoundingClientRect` gives one. */ +export type Box = { left: number; top: number; right: number; bottom: number; width: number; height: number }; + +/** The window a popover has to fit into. */ +export type View = { width: number; height: number }; + +/** Where a popover ends up, and how tall it may be before it has to scroll. */ +export type Placement = { left: number; top: number; maxHeight: number }; + +/** How close a popover may come to the edge of the window. */ +export const MARGIN = 8; + +/** How far a popover sits from the thing it hangs off. */ +const GAP = 6; + +const clamp = (value: number, low: number, high: number) => Math.max(low, Math.min(value, high)); + +/** + * Fits a popover that hangs below something: the element menu under the element it is about. + * + * Below when the window has room for it there, above when it does not, and pinned to the edge it + * ran past when neither side fits — a popover against the bottom of the window still shows its + * rows, where one left below the fold shows none. It is free to lie over whatever is behind it. + */ +export function placeBelow(anchor: Box, size: { width: number; height: number }, view: View): Placement { + const maxHeight = Math.max(120, view.height - MARGIN * 2); + const height = Math.min(size.height, maxHeight); + const below = anchor.bottom + GAP; + const above = anchor.top - GAP - height; + const top = below + height <= view.height - MARGIN ? below + : above >= MARGIN ? above + : clamp(view.height - MARGIN - height, MARGIN, Math.max(MARGIN, view.height - MARGIN - height)); + return { + left: clamp(anchor.left, MARGIN, Math.max(MARGIN, view.width - size.width - MARGIN)), + top, + maxHeight, + }; +} + +/** + * Fits a submenu beside the row it hangs off. + * + * It always opens to the right, which is the side its chevron points at. There is no flip to the + * left: the panel this runs in is often only a few hundred points wide, so "flip when it does not + * fit" meant it opened on the left nearly always — pointing one way and opening the other. When + * the right has no room the submenu is pushed in from the window's edge instead, overlapping the + * menu it came from, which is what a cascading menu does in a tight window. + * + * Its bottom lines up with its row and it grows upward, since the row is the last in the menu. + * + * `flip` opts into opening on the left when the right has no room *and the left does*, which is + * what a cascading branch several levels deep needs: by the third panel the right edge is reached + * on any ordinary window, and overlapping the menu it came from hides the trail the user is + * following. It stays opt-in because the narrow case above is still real — when neither side fits, + * flipping would point one way and open the other, so the pushed-in placement is what happens. + * + * `align` is which of the row's edges the panel lines up with: `bottom` for a row at the end of a + * menu, `top` for one in the middle of a list, where growing upward would hide the rows above it. + * + * `side: "left"` asks for the left outright, whether or not the right has room, for a branch whose + * parent menu is itself against the right edge of the workspace. It falls back to the right only + * when the left cannot hold the panel at all. + */ +export function placeBeside( + row: Box, + size: { width: number; height: number }, + view: View, + options: { flip?: boolean; align?: "top" | "bottom"; side?: "left" | "right" } = {}, +): Placement { + const maxHeight = Math.max(120, view.height - MARGIN * 2); + const height = Math.min(size.height, maxHeight); + const rightmost = Math.max(MARGIN, view.width - size.width - MARGIN); + const right = row.right + 4; + const left = row.left - 4 - size.width; + // `side: "left"` is a branch that opens on the left by design rather than because the right ran + // out: the step list is the rightmost thing in the panel, so a submenu of a step row opens over + // the preview — which is where there is room — instead of over the list it came from. It still + // falls back to the right when the left has no room, because a submenu off the edge of the + // window shows nothing at all. + const chooseLeft = options.side === "left" + ? left >= MARGIN + : options.flip && right > rightmost && left >= MARGIN; + return { + left: chooseLeft ? left : clamp(right, MARGIN, rightmost), + // Lined up with the chosen edge, then pushed back in if that would take it off either edge. + top: clamp(options.align === "top" ? row.top : row.bottom - height, MARGIN, + Math.max(MARGIN, view.height - MARGIN - height)), + maxHeight, + }; +} diff --git a/packages/ui/src/simulator-panel.ts b/packages/ui/src/simulator-panel.ts index 168bb7b9..5f76706c 100644 --- a/packages/ui/src/simulator-panel.ts +++ b/packages/ui/src/simulator-panel.ts @@ -18,8 +18,9 @@ import { captureUiError, countUiEvent, recordUiTiming } from "./telemetry.ts"; import type { PanelContext } from "./model-context.ts"; import { encodeScreenshotAttachment } from "./screenshot-attachment.ts"; import { getScreenAnnotations } from "./screen-annotations.ts"; +import { testRecorder } from "./test-recorder.ts"; import type { ScreenAnnotation } from "@mobile-dev/shared/screen-annotations.ts"; -import { accessibilityScreen, screenComponents, screenSize } from "@mobile-dev/shared/screen-annotations.ts"; +import { accessibilityScreen, scaledComponents, screenComponents, screenSize } from "@mobile-dev/shared/screen-annotations.ts"; import { AndroidVideo } from "./android-video.ts"; import { PhysicalIosVideo } from "./ios-mirror-video.ts"; import { iosVideoBatchSchema } from "@mobile-dev/shared/ios-video.ts"; @@ -32,6 +33,7 @@ import type { StreamMessage } from "./stream-input.ts"; import { IosVideo } from "./ios-video.ts"; import type { AndroidVideoBatch } from "./android-video.ts"; import type { HostBridge } from "./host-bridge.ts"; +import { isRecordableDevice, recordablePlatform } from "@mobile-dev/shared/panel-target.ts"; import type { PanelTarget } from "@mobile-dev/shared/panel-target.ts"; import { initialPanelDevice } from "./device-layout.ts"; @@ -140,6 +142,9 @@ export function createSimulatorPanel( function send(message: StreamMessage) { if (!ready || !stream || inputBlocked) return; + // Every input can change the screen, so a tree read before it no longer describes what the + // next tap lands on. Typing and the hardware buttons go through here too, not just touches. + if (isRecordingSelected()) testRecorder.invalidate(); stream.input?.send(message); } @@ -278,6 +283,54 @@ export function createSimulatorPanel( } } finally { recordUiTiming("ui.annotations.inspection", performance.now() - startedAt); } }; + /** + * The Testing panel records from the device this panel shows, and this is how it reads it: not + * the inspection path the annotation overlay uses, but the e2e runner's own element tree, so a + * recorded selector resolves to the same element when the test runs. The Testing panel installs + * that source — it knows the project and the target — and this supplies the device and the + * screen. Only iOS simulators and Android emulators are wired up; a physical device has no tree here at all. + */ + const recorderReader = { + readComponents: async () => { + if (!selected) return []; + const { tree, screen } = await testRecorder.readTree(selected.udid, recordablePlatform(selected)); + // `screen` is the space the runner measured in; `points` is the space the mirror maps + // pointer events into. They agree on an iOS simulator and do not on Android, where scrcpy + // caps the stream's longest edge while the tree stays in device pixels. + return screen ? scaledComponents(tree, screen, points) : tree; + }, + screen: () => points, + release: () => testRecorder.releaseTree(), + // Taking the step the menu recorded. `tap` and `type` are the panel's own input messages, so + // the app advances exactly as a user's own tap would, and the next step is resolved against + // the screen the test will be on. + perform: (action: + | { kind: "tap"; point: { x: number; y: number } } + | { kind: "type"; text: string } + | { kind: "swipe"; from: { x: number; y: number }; to: { x: number; y: number } }) => { + if (action.kind === "tap") send({ type: "tap", x: action.point.x, y: action.point.y, ...points }); + // The stream carries a swipe of its own, so a recorded `swipe` is taken as the gesture it + // emits rather than merely written down. `duration` is in seconds here, and a third of one + // is a deliberate drag: fast enough not to be read as a long press, slow enough that the + // app sees the intermediate touches rather than a jump. + else if (action.kind === "swipe") { + send({ type: "swipe", startX: action.from.x, startY: action.from.y, endX: action.to.x, endY: action.to.y, ...points, duration: 0.3 }); + } + // The runner's `fill` replaces the field's contents; typing on the mirror appends to them, and + // there is no input message that clears a field. So a `fill` recorded into a field that + // already has text leaves the app showing something the test will not produce — the step is + // still right, the screen behind it is not. Non-ASCII is refused by the input schema. + else if (action.text) send({ type: "type", text: action.text }); + }, + }; + function isRecordingSelected() { + return selected ? testRecorder.isRecording(selected.udid, recordablePlatform(selected)) : false; + } + function syncRecorder() { + if (selected && isRecordableDevice(selected)) testRecorder.attach(selected.udid, recordablePlatform(selected), recorderReader); + else if (selected) testRecorder.detach(selected.udid, recordablePlatform(selected)); + } + const stopObservingAnnotations = annotations.subscribe(() => { const state = annotations.getSnapshot(); canvas.dataset.selecting = String(state.selecting || !!state.draft); @@ -455,6 +508,7 @@ export function createSimulatorPanel( selected = status?.devices.find(device => device.udid === devices.value); if (selected === undefined) selected = physicalDevices.find(device => device.udid === devices.value); selectionChanged(selected); + syncRecorder(); if (selected?.kind === "physical" && selected.platform === "ios") { const connection = physicalConnectionLabel(selected.transportType); if (reconnect.active === false) empty(`${selected.name} · ${connection}`, "Opening the physical device screen…"); @@ -889,20 +943,72 @@ export function createSimulatorPanel( return; } if (annotations.getSnapshot().draft) return; + // A click outside the element menu dismisses it rather than recording through it; the menu + // stops its own pointer events from reaching the canvas at all. + if (testRecorder.getSnapshot().menu) { testRecorder.closeMenu(); return; } + // While an agent is recording, the mirror is its screen: a tap landing between the agent's + // read and its step would resolve that step against a screen that has already moved. Two + // drivers on one screen is solved by locking, not by hoping. + if (testRecorder.isAgentRecording(selected?.udid)) return; if (!ready || inputBlocked || busy || event.button !== 0 || pointer) return; const position = mappedPoint(event); - const edge = position.y > points.height - 14 ? "bottom" : position.y < 14 ? "top" : undefined; + const edge: TouchInput["edge"] = position.y > points.height - 14 ? "bottom" : position.y < 14 ? "top" : undefined; + const touch = { ...position, ...points, ...(edge ? { edge } : {}) }; + // A left click only drives the app, recording or not. Recording a step from it meant every + // stray tap while finding the next screen became a step the user then had to delete, and it + // made "tap to drive" and "tap to record" the same gesture. Steps come from the element menu. pointer = { id: event.pointerId, ...position, edge }; canvas.setPointerCapture(event.pointerId); canvas.focus(); - send({ type: "touch1-down", ...position, ...points, ...(edge ? { edge } : {}) }); + send({ type: "touch1-down", ...touch }); }); canvas.addEventListener("pointermove", event => { if (annotations.getSnapshot().selecting) { if (annotationPointer === undefined || annotationPointer === event.pointerId) annotations.hover(mappedPoint(event)); return; } + // While recording, the pointer previews what it would record. Not during a drag: the touch is + // already going to the device, and the screen under it is no longer the one that was read. + if (testRecorder.isAgentRecording(selected?.udid)) return; + if (isRecordingSelected() && !pointer) { void testRecorder.hover(mappedPoint(event)); return; } if (!pointer || pointer.id !== event.pointerId) return; Object.assign(pointer, mappedPoint(event)); send({ type: "touch1-move", x: pointer.x, y: pointer.y, ...points, ...(pointer.edge ? { edge: pointer.edge } : {}) }); }); - canvas.addEventListener("pointerleave", () => { if (annotationPointer === undefined) annotations.hover(); }); + canvas.addEventListener("pointerleave", () => { if (annotationPointer === undefined) annotations.hover(); void testRecorder.hover(); }); + // Right-click asks what can be done with the element the outline is on, instead of the browser's + // own menu over a canvas that has nothing in it to copy. The menu itself is drawn by the + // recorder overlay, which already has the outline's rectangle to anchor to. + // Escape closes the element menu. On the window and in the capture phase, so it beats the + // canvas's own key handling, which would otherwise forward the key to the device. + window.addEventListener("keydown", event => { + if (disposed || event.key !== "Escape" || !testRecorder.getSnapshot().menu) return; + event.preventDefault(); + event.stopPropagation(); + testRecorder.closeMenu(); + }, { capture: true }); + canvas.addEventListener("contextmenu", event => { + if (!isRecordingSelected() || annotations.getSnapshot().selecting) return; + event.preventDefault(); + // The element menu records a step and takes it, which is exactly the second driver the lock + // exists to stop. The browser's own menu is still suppressed, so the lock is not a regression + // to a canvas context menu with nothing in it. + if (testRecorder.isAgentRecording(selected?.udid)) return; + void testRecorder.openMenu(mappedPoint(event)); + }); + // While recording, the wheel walks the outline out to the enclosing element and back in — the + // label inside the row, the row inside the card. The mirror has no other use for the wheel. + // A trackpad sends a burst of small deltas per flick, so steps are accumulated rather than + // taken per event — otherwise one flick walks straight out to the root. + let wheelTravel = 0; + const WHEEL_STEP = 60; + canvas.addEventListener("wheel", event => { + if (!isRecordingSelected() || annotations.getSnapshot().selecting) return; + event.preventDefault(); + if (testRecorder.isAgentRecording(selected?.udid)) return; + if (wheelTravel !== 0 && Math.sign(event.deltaY) !== Math.sign(wheelTravel)) wheelTravel = 0; + wheelTravel += event.deltaY; + while (Math.abs(wheelTravel) >= WHEEL_STEP) { + testRecorder.walkHover(wheelTravel > 0); + wheelTravel -= Math.sign(wheelTravel) * WHEEL_STEP; + } + }, { passive: false }); canvas.addEventListener("pointerup", event => { if (annotationPointer === event.pointerId) { annotations.endSelection(mappedPoint(event)); diff --git a/packages/ui/src/stored-test-steps.ts b/packages/ui/src/stored-test-steps.ts new file mode 100644 index 00000000..8301a855 --- /dev/null +++ b/packages/ui/src/stored-test-steps.ts @@ -0,0 +1,102 @@ +/** A saved test has source but no recorder step data; edit its awaited call lines in a working copy. */ +export type StoredTestStep = { line: number; label: string }; + +/** Every awaited call line, including app.open, in the order the runner indexes them. */ +export function storedTestCallLines(source: string): number[] { + return source.split("\n").flatMap((line, index) => /^\s*await\b/.test(line) ? [index + 1] : []); +} + +export function storedTestSteps(source: string): StoredTestStep[] { + return source.split("\n").flatMap((line, index) => { + const expression = line.match(/^\s*await\s+(.+?)\s*;?\s*$/)?.[1]; + if (!expression || /^app\.open\s*\(/.test(expression)) return []; + return [{ line: index + 1, label: expression.replace(/;$/, "") }]; + }); +} + +function editLine(source: string, line: number, replace: (lines: string[], index: number) => void): string { + const lines = source.split("\n"); + if (!storedTestSteps(source).some(step => step.line === line) || !storedTestUntilSource(source, line)) return source; + replace(lines, line - 1); + return lines.join("\n"); +} + +export function duplicateStoredStep(source: string, line: number): string { + return editLine(source, line, (lines, index) => { lines.splice(index + 1, 0, lines[index]!); }); +} + +export function removeStoredStep(source: string, line: number): string { + return editLine(source, line, (lines, index) => { lines.splice(index, 1); }); +} + +export function moveStoredStep(source: string, fromLine: number, toLine: number): string { + if (fromLine === toLine || !storedTestSteps(source).some(step => step.line === toLine)) return source; + return editLine(source, fromLine, (lines, index) => { + const [moved] = lines.splice(index, 1); + lines.splice(toLine - 1, 0, moved!); + }); +} + +export type StoredCommand = { id: string; label: string; expression: string }; + +/** Only commands whose locator can be recovered without guessing are offered. */ +export function storedStepCommands(step: StoredTestStep): StoredCommand[] { + const assertion = step.label.match(/^expect\((.+)\)\.(?:toBeVisible|not\.toBeVisible|toBeEnabled|toBeDisabled|toBeSelected)\((?:\{\s*timeout:\s*\d+\s*\})?\)$/); + const action = step.label.match(/^(.+)\.(?:tap|doubleTap|longPress)\(\)$/); + const scroll = step.label.match(/^screen\.scrollUntilVisible\((.+)\)$/); + const locator = assertion?.[1] ?? action?.[1] ?? scroll?.[1]; + if (!locator) return []; + return [ + { id: "tap", label: "Tap", expression: `${locator}.tap()` }, + { id: "doubleTap", label: "Double tap", expression: `${locator}.doubleTap()` }, + { id: "longPress", label: "Long press", expression: `${locator}.longPress()` }, + { id: "scrollUntilVisible", label: "Scroll into view", expression: `screen.scrollUntilVisible(${locator})` }, + { id: "assertVisible", label: "Is visible", expression: `expect(${locator}).toBeVisible()` }, + { id: "assertNotVisible", label: "Is not visible", expression: `expect(${locator}).not.toBeVisible()` }, + { id: "assertEnabled", label: "Is enabled", expression: `expect(${locator}).toBeEnabled()` }, + { id: "assertDisabled", label: "Is disabled", expression: `expect(${locator}).toBeDisabled()` }, + { id: "assertSelected", label: "Is selected", expression: `expect(${locator}).toBeSelected()` }, + ]; +} + +export function changeStoredStepCommand(source: string, line: number, commandId: string): string { + const step = storedTestSteps(source).find(entry => entry.line === line); + const command = step && storedStepCommands(step).find(option => option.id === commandId); + if (!command) return source; + const changed = editLine(source, line, (lines, index) => { + const indent = lines[index]!.match(/^\s*/)?.[0] ?? ""; + lines[index] = `${indent}await ${command.expression};`; + }); + if (!commandId.startsWith("assert") || /import\s*{[^}]*\bexpect\b[^}]*}\s*from\s*["']e2e["']/.test(changed)) return changed; + const lines = changed.split("\n"); + const lastImport = lines.findLastIndex(entry => /^import\s/.test(entry)); + lines.splice(lastImport + 1, 0, 'import { expect } from "e2e";'); + return lines.join("\n"); +} + +/** A complete one-line await can be moved safely; a multi-line call stays in the source editor. */ +function completeAwaitLine(line: string): boolean { + const expression = line.match(/^\s*await\s+(.+?);?\s*$/)?.[1]; + if (!expression || /[,(.?:=+\-*/{}\[]$/.test(expression.trimEnd())) return false; + const closing: Record<string, string> = { "(": ")", "[": "]", "{": "}" }; + const stack: string[] = []; + let quote = ""; + for (let index = 0; index < expression.length; index++) { + const char = expression[index]!; + if (char === "\\" && quote) { index++; continue; } + if (quote) { if (char === quote) quote = ""; continue; } + if (char === '"' || char === "'" || char === "`") { quote = char; continue; } + if (closing[char]) stack.push(closing[char]); + else if ([")", "]", "}"].includes(char) && stack.pop() !== char) return false; + } + return !quote && stack.length === 0; +} + +/** The runner uses a temporary sibling file; returning here stops after the chosen call. */ +export function storedTestUntilSource(source: string, line: number): string | undefined { + const lines = source.split("\n"); + if (!completeAwaitLine(lines[line - 1] ?? "")) return; + const indent = lines[line - 1]!.match(/^\s*/)?.[0] ?? ""; + lines.splice(line, 0, `${indent}return;`); + return lines.join("\n"); +} diff --git a/packages/ui/src/style.css b/packages/ui/src/style.css index 0d761006..bc7da234 100644 --- a/packages/ui/src/style.css +++ b/packages/ui/src/style.css @@ -362,3 +362,252 @@ html[data-theme="dark"] .tracesift-explorer { color-scheme: dark; } @media (prefers-reduced-motion: reduce) { .tracesift-upload .upload-pane { transition: none; } } + +/* + * The recorded test's preview. A step is added by tapping the mirror, where the user is looking — + * not at the preview — so the new lines announce themselves: they settle in, hold a tint long + * enough to be caught out of the corner of the eye, and fade back into the rest of the file. + */ +.testing-preview-line { + display: grid; + /* The line number, the gutter, then the code. All fixed, so nothing moves when a run marks a + line: the number column is wide enough for a three-figure file and does not grow at 100. */ + grid-template-columns: 3ch 1.35em minmax(0, 1fr); + align-items: start; + min-height: 1lh; + border-radius: 3px; + margin-inline: -2px; + padding-inline: 2px; + /* Air between one line and the next. The file is read down a narrow pane beside the mirror, and + a wrapped call run together with the one below it is the hardest thing in the panel to scan. + It is padding rather than a margin so the hover highlight and the arrival tint cover it too, + and `.testing-play-until` offsets by the same amount to stay level with its line. */ + padding-block: 2px; + /* The hover highlight is painted as a layer over the line; see below. */ + position: relative; +} +/* + * The line numbers. Out of the selection, because the preview is there to be copied into a file: + * dragging over it has to give the code and nothing else. `tabular-nums` keeps the column from + * shifting under the eye as a 1 follows a 0 down the file. + */ +.testing-preview-number { + height: 1lh; + padding-inline-end: 0.5ch; + color: var(--muted-foreground); + opacity: 0.55; + text-align: right; + font-variant-numeric: tabular-nums; + user-select: none; + -webkit-user-select: none; +} +/* The line's own text wraps inside its column; the gutter is not part of the wrap. */ +.testing-preview-code { white-space: pre-wrap; overflow-wrap: anywhere; } + +/* The editable layer shares the preview tokenizer. Its transparent text leaves the highlighted + code visible while the native textarea supplies selection, a caret, input and scrolling. */ +.testing-source-editor-code { + position: relative; + min-height: 0; + flex: 1; + overflow: hidden; + border: 1px solid var(--border); + border-radius: 0.375rem; + font-family: var(--font-mono), monospace; + font-size: 11px; + line-height: 1.625; + tab-size: 2; +} +.testing-source-editor-code:focus-within { outline: 2px solid color-mix(in oklab, var(--ring) 50%, transparent); outline-offset: -2px; } +.testing-source-editor-code > pre, +.testing-source-editor-input { + position: absolute; + inset: 0; + box-sizing: border-box; + width: 100%; + height: 100%; + margin: 0; + border: 0; + padding: 0.5rem; + font: inherit; + line-height: inherit; + tab-size: inherit; + white-space: pre; +} +.testing-source-editor-code > pre { overflow: hidden; pointer-events: none; } +.testing-source-editor-line { display: block; min-height: 1lh; } +.testing-source-editor-number { + display: inline-block; + width: 3ch; + margin-right: 2ch; + color: var(--muted-foreground); + opacity: 0.55; + text-align: right; + user-select: none; +} +.testing-source-editor-input { + overflow: auto; + resize: none; + padding-left: calc(0.5rem + 5ch); + background: transparent; + color: transparent; + caret-color: var(--foreground); + outline: none; +} +.testing-source-editor-input::selection { background: color-mix(in oklab, var(--primary) 25%, transparent); } +.testing-source-suggestions { + position: absolute; + z-index: 3; + display: flex; + width: min(240px, calc(100% - 12px)); + max-height: 168px; + flex-direction: column; + overflow: auto; + border: 1px solid var(--border); + border-radius: 0.375rem; + background: var(--popover); + padding: 4px; + color: var(--popover-foreground); + box-shadow: 0 4px 12px rgb(0 0 0 / 16%); +} +.testing-source-suggestion { + display: block; + width: 100%; + border-radius: 0.25rem; + padding: 4px 6px; + text-align: left; + white-space: nowrap; + cursor: pointer; +} +.testing-source-suggestion[aria-selected="true"], +.testing-source-suggestion:hover { background: var(--accent); color: var(--accent-foreground); } +.testing-preview-line.is-new { animation: testing-preview-line-in 1400ms ease-out both; } +/* + * The hover highlight: a layer of its own, not the line's own `background`. + * + * It used to be a background plus `animation: none`, which was a bug with a long tail. The arrival + * animation above holds its end state on the lines a step added, and those lines keep `is-new` for + * as long as the file does not change — so suppressing the animation while a step was hovered + * *cancelled* it, and letting it back on mouse-leave **restarted it from the beginning**. Leaving + * a step lit its lines up again and faded them out over 1400ms, which is what a highlight must + * never do: it has to go the moment the pointer does. + * + * As a layer there is nothing to suppress. The animation keeps running underneath, the highlight + * appears and disappears with the class, and the two never touch the same property. + */ +.testing-preview-line.is-step-highlighted::before { + content: ""; + position: absolute; + inset: 0; + border-radius: 3px; + background: color-mix(in oklab, var(--primary) 16%, transparent); + box-shadow: inset 2px 0 var(--primary); + pointer-events: none; +} +/* Positioned too, so the number, the code and the run mark paint above that layer rather than + under it. */ +.testing-preview-number, .testing-run-mark, .testing-preview-code { position: relative; } +@keyframes testing-preview-line-in { + from { background: color-mix(in oklab, var(--primary) 26%, transparent); opacity: 0.4; } + 18% { background: color-mix(in oklab, var(--primary) 26%, transparent); opacity: 1; } + 60% { background: color-mix(in oklab, var(--primary) 20%, transparent); } + to { background: transparent; opacity: 1; } +} + +@media (prefers-reduced-motion: reduce) { + /* No movement, but the line still has to be findable, so the tint is simply held and dropped. */ + .testing-preview-line.is-new { animation: testing-preview-line-hold 1400ms steps(1, end) both; } + @keyframes testing-preview-line-hold { + from { background: color-mix(in oklab, var(--primary) 20%, transparent); } + to { background: transparent; } + } +} + +/* + * A run's per-line marks. The mark occupies a fixed gutter whether or not it is showing, so the + * code never shifts sideways when a run starts or ends — the preview is read while the run is + * happening, and text that moves under the eye is worse than no mark at all. + * + * A passing run holds its checkmarks and then fades them: the test is the thing on screen, not the + * scoreboard. A failure keeps its mark until the next run. + */ +.testing-run-mark { + display: flex; + align-items: center; + justify-content: center; + /* Exactly one line box tall, so the mark centres on the line it belongs to rather than drifting + with the icon's own box — and so a wrapped line keeps its mark level with the first row. */ + height: 1lh; + opacity: 0; + transition: opacity 240ms ease-out; +} +.testing-run-mark.is-running { opacity: 1; color: var(--muted-foreground); } +.testing-run-mark.is-passed { opacity: 1; color: light-dark(oklch(0.6 0.13 157), oklch(0.77 0.14 157)); } +.testing-run-mark.is-failed { opacity: 1; color: light-dark(oklch(0.58 0.2 25), oklch(0.7 0.19 25)); } +/* Only ever set on a passing tick; a failing mark is never given this class. */ +.testing-run-mark.is-fading { opacity: 0; transition-duration: 600ms; } + +@media (prefers-reduced-motion: reduce) { + /* The marks still come and go; they just stop easing, so nothing animates on screen. */ + .testing-run-mark { transition: none; } + .testing-run-mark.is-fading { transition: none; } +} + +/* + * Play until here: the gutter's other occupant. + * + * It sits *over* the gutter rather than in it, so the column stays exactly as wide as it was and + * no line of code moves when a play appears — the same reason the mark has a fixed column at all. + * Pointing at a line swaps its mark for the play, and the mark comes back when the pointer leaves; + * the two never show together, because a gutter that says both "passed" and "run to here" says + * neither. Nothing is rendered at all while a run is in flight, so a spinner is never hidden by a + * pointer resting on its line. + */ +.testing-play-until { + position: absolute; + /* The line's own padding, so this lands on the gutter column rather than beside it, and level + with the mark it stands in for rather than 2px above it. See `padding-block` above. */ + inset-inline-start: calc(2px + 3ch); + top: 2px; + display: flex; + width: 1.35em; + height: 1lh; + align-items: center; + justify-content: center; + border-radius: 3px; + color: var(--muted-foreground); + opacity: 0; + cursor: pointer; + transition: opacity 120ms ease-out, color 120ms ease-out; + /* Hidden until the line is pointed at, and unreachable by the pointer until then: an invisible + button that still swallows clicks is worse than no button. */ + pointer-events: none; +} +.testing-preview-line:hover .testing-play-until, +.testing-play-until:focus-visible { + opacity: 1; + pointer-events: auto; +} +.testing-play-until:hover, .testing-play-until:focus-visible { color: var(--primary); } +/* Saved project tests keep the play control beside the run mark without showing it until hover. */ +[data-testing-stored-preview] .testing-preview-line { grid-template-columns: 3ch 1.35em 1.35em minmax(0, 1fr); } +[data-testing-stored-preview] .testing-preview-code { grid-column: 4; } +[data-testing-stored-preview] .testing-play-until { + position: relative; + grid-column: 3; + inset: auto; +} +[data-testing-stored-preview] .testing-preview-line:hover .testing-play-until, +[data-testing-stored-preview] .testing-play-until:focus-visible { + opacity: 1; + pointer-events: auto; +} +/* The mark steps aside for the play, and only on a line that has one. */ +.testing-preview-line:hover:has(.testing-play-until) .testing-run-mark, +.testing-preview-line:has(.testing-play-until:focus-visible) .testing-run-mark { opacity: 0; } +[data-testing-stored-preview] .testing-preview-line:hover:has(.testing-play-until) .testing-run-mark, +[data-testing-stored-preview] .testing-preview-line:has(.testing-play-until:focus-visible) .testing-run-mark { opacity: 1; } + +@media (prefers-reduced-motion: reduce) { + .testing-play-until { transition: none; } +} diff --git a/packages/ui/src/telemetry.ts b/packages/ui/src/telemetry.ts index 6cd0b1e0..03a1c4cb 100644 --- a/packages/ui/src/telemetry.ts +++ b/packages/ui/src/telemetry.ts @@ -30,12 +30,17 @@ function cancelReadinessFrames() { readinessFrames.clear(); } -export function recordUiTiming(name: string, duration: number) { +/** + * `platform` splits one timing into a series per mirrored platform (two values, so still low + * cardinality): the window is keyed by name and platform and flushed with the attribute. + */ +export function recordUiTiming(name: string, duration: number, platform?: "ios" | "android") { if (running === false || document.visibilityState === "hidden") return; - let window = measurements.get(name); + const key = platform ? `${name}\u0000${platform}` : name; + let window = measurements.get(key); if (window === undefined) { window = new MeasurementWindow(); - measurements.set(name, window); + measurements.set(key, window); } window.record(duration); } @@ -102,13 +107,15 @@ export function flushUiMeasurements() { if (running === false) return; const now = performance.now(); recordVisibleTime(now); - for (const [name, window] of measurements) { + for (const [key, window] of measurements) { const summary = window.take(); if (summary === undefined) continue; - Sentry.metrics.count(`${name}.samples`, summary.count, { attributes }); - Sentry.metrics.gauge(`${name}.mean`, summary.mean, { unit: "millisecond", attributes }); - Sentry.metrics.gauge(`${name}.p95`, summary.p95, { unit: "millisecond", attributes }); - Sentry.metrics.gauge(`${name}.max`, summary.max, { unit: "millisecond", attributes }); + const [name, platform] = key.split("\u0000"); + const scoped = platform ? { ...attributes, platform } : attributes; + Sentry.metrics.count(`${name}.samples`, summary.count, { attributes: scoped }); + Sentry.metrics.gauge(`${name}.mean`, summary.mean, { unit: "millisecond", attributes: scoped }); + Sentry.metrics.gauge(`${name}.p95`, summary.p95, { unit: "millisecond", attributes: scoped }); + Sentry.metrics.gauge(`${name}.max`, summary.max, { unit: "millisecond", attributes: scoped }); } for (const [name, value] of gauges) Sentry.metrics.gauge(name, value, { attributes }); for (const [name, value] of counts) Sentry.metrics.count(name, value, { attributes }); diff --git a/packages/ui/src/test-recorder.ts b/packages/ui/src/test-recorder.ts new file mode 100644 index 00000000..0e4f8bae --- /dev/null +++ b/packages/ui/src/test-recorder.ts @@ -0,0 +1,1026 @@ +import { agentStepLabel, allStepOffers, anchorSelector, emitE2eTest, emitE2eTestLines, hitLadder, isAgentStep, resolveSelector, stepEffect, stepIsAssertion, stepLabel, stepNeedsValue, stepOffers, swipeDirection, swipePath, treeHealth } from "@mobile-dev/shared/test-recording.ts"; +import type { AgentParams, AgentStep, AgentStepKind, PinnedStep, PinnedStepKind, RecordedStep, StepOffer, StepTarget, SwipeDirection, VisionMode } from "@mobile-dev/shared/test-recording.ts"; +import type { ScreenBounds, ScreenComponent, ScreenPoint, ScreenSize } from "@mobile-dev/shared/screen-annotations.ts"; +import { countUiEvent, recordUiTiming, setUiGauge } from "./telemetry.ts"; + +/** + * Reads the screen the recorder is recording against. The simulator panel owns the device + * connection, so it supplies this; the recorder never talks to the host itself. + */ +export type RecorderReader = { + /** The runner's element tree as flat components, in the same coordinate space as `screen`. */ + readComponents: () => Promise<ScreenComponent[]>; + /** The screen size the mirror maps pointer events into. */ + screen: () => ScreenSize; + /** + * Drives the device for a step the menu just recorded. + * + * A step that is recorded but never taken leaves the app on the screen before it, so the next + * row is resolved against a screen the test will not be on. The recorder does not touch the + * device itself — the simulator panel owns the stream — so it asks. + */ + perform?: (action: + | { kind: "tap"; point: ScreenPoint } + | { kind: "type"; text: string } + // Start and end in screen space, as every other point here is; the panel's own stream carries + // a swipe, so the gesture a recorded step emits is one the mirror can also take. + | { kind: "swipe"; from: ScreenPoint; to: ScreenPoint }) => void; + /** + * Gives the device back to whoever wants it next. + * + * Reading the runner's tree holds a session on the device, and the runner allows exactly one, so + * a session left open here makes the user's next `e2e run` fail with "device is already in use" + * — a refusal they cannot connect to anything they did. The recorder calls this on every path + * out of recording: stop, detach, and a device that changes under it. + */ + release?: () => void; +}; + +/** + * Reads the runner's element tree for one device, and owns the session that read needs. + * + * The Testing panel installs this, because it is the panel that knows which e2e project and which + * target the recording is for; the simulator panel knows the device and the screen. The recorder + * is the one object both already share, so it is where the two halves meet. + */ +/** A device is the pair: an iOS UDID and an adb serial are different namespaces. */ +export type DevicePlatform = "ios" | "android"; + +export type RecorderTreeSource = { + /** + * The runner's tree, and the screen its rectangles are measured against. + * + * The screen is the runner's, not the mirror's: on Android the two differ, and the panel that + * owns the mirror converts between them. See `scaledComponents`. + */ + read: (deviceId: string, platform: DevicePlatform) => Promise<{ tree: ScreenComponent[]; screen?: ScreenSize }>; + release: () => void; +}; + +/** + * The `app.open()` the emitter writes above the steps, as Play until here names it. + * + * It is not one of the recorded steps — it is the header's own call — so it has no step id to go + * by, and this stands in for one. It is not a UUID, which is what `crypto.randomUUID` gives every + * recorded step, so it can never be mistaken for one of them. + */ +export const OPEN_STEP = "app:open"; + +/** What a run marks one line of the preview with. */ +export type RunLineMark = "running" | "passed" | "failed"; + +/** + * A run, as the preview shows it. + * + * The marks are keyed by 1-based line rather than by step because that is what the runner's report + * gives: every step it executed carries the line it came from. `state` is the file's own verdict. + */ +export type TestRunView = { + state: "running" | "passed" | "failed"; + lines: Record<number, RunLineMark>; + /** The first failure's message, from the report, for the line that failed. */ + error?: string; + /** The runner said nothing a line could be marked with — its own output is the explanation. */ + output?: string; + /** + * How long the run took, from the report. Not the panel's own `ui.testing.run` timing, which + * brackets the whole call including spawning the CLI; see `reportDuration`. + */ + durationMs?: number; + /** + * The passing ticks are on their way out; the CSS honours `prefers-reduced-motion`. Only they + * fade — a failing mark is the one thing a failed run leaves behind to look at. + */ + fading?: boolean; + /** + * The run was a Play until here: it ran the flow only as far as this line, so the lines below it + * were never reached. A verdict about part of a test must not read as a verdict about the test. + */ + untilLine?: number; + /** + * The steps changed after the run, so the line marks have been dropped: they were claims about + * lines that have since moved. The verdict and the duration stay — they are facts about a run + * that happened, and they do not stop being true because the next flow is being written. + */ + stale?: true; +}; + +export type TestRecorderSnapshot = { + /** The device being recorded against (UDID or adb serial), or "" when nothing is recording. */ + recordingDeviceId: string; + /** Its platform, or "" when nothing is recording. */ + recordingPlatform: DevicePlatform | ""; + /** + * The device an *agent* is recording against, or "" when the user owns the flow. + * + * The steps, the preview, the emitter and Save are the same ones either way — this is + * deliberately not a second state machine — so the only thing this answers is who is driving. + * The mirror's input is locked while it is set, because the agent and the user must not both be + * acting on one screen, and the panel draws its banner from it. + */ + agentDeviceId: string; + steps: RecordedStep[]; + /** Source edited directly in the preview; supersedes the generated source. */ + sourceOverride?: string; + title: string; + /** Bumped by every change to what would be written, so the panel can tell saved from dirty. */ + revision: number; + /** + * Which flow the step list *is*, as opposed to how many times it has changed. + * + * A new one is minted whenever the list stops being the same recording — Record on a flow the + * user did not choose to resume, Clear, a kept recording reopened, an agent session beginning. + * `revision` cannot answer this: it moves for an edit and for a brand new flow alike, and the + * panel has to tell those apart or it keeps writing one recording over another's file. See + * `TestingPanel.keepDraft`. + */ + flowId: string; + /** A step is being resolved: the tap is held until its tree read finishes. */ + resolving: boolean; + /** + * Recording has been asked for but the screen cannot be named yet. + * + * Pressing Record arms the recorder immediately, but nothing on the mirror can be selected until + * the runner's element tree has been read — attaching to the app and projecting the tree takes a + * second or two — and until this existed the panel said "Record" and then ignored the first few + * seconds of pointing, with nothing on screen to say why. `start` now warms the tree itself and + * holds this until that read lands, which is the moment selection actually works. + */ + preparing: boolean; + /** + * What a tap at the pointer would record, for the outline drawn over the mirror: the bounds of + * the element the selector would name, the step it would add, and the screen those bounds are + * in so the overlay can scale them to the canvas. + */ + hover?: { + bounds: ScreenBounds; label: string; screen: ScreenSize; ambiguous: boolean; + /** How far out the user has walked, and how far out they still can. */ + level: number; levels: number; + }; + /** + * The element menu a right-click opened: what the runner can do with the element the outline is + * on, each row carrying the action and emitted code it represents. Walking the outline out with the wheel and then + * right-clicking acts on the element the outline shows, which is why the menu resolves at the + * hover level rather than from the innermost hit. + */ + menu?: { + label: string; + ambiguous: boolean; + /** How many elements the offered selector matches; 1 unless the menu is flagged. */ + matches: number; + offers: StepOffer[]; + /** + * Every step the runner can express for the same element, for the menu's "all commands" list. + * The short list above is what most steps are; this is the rest of the runner's surface, so a + * step it has — an element that should be *gone*, a field's value, a disabled button — does + * not need the user to leave the panel and write the line themselves. + */ + commands: StepOffer[]; + /** + * What the element carries, for the Agentic rows to write their first draft out of. The + * instruction is free text the user edits and the runner resolves itself, so this only ever + * saves them typing; see `agentActSuggestion`. + */ + target?: StepTarget; + /** + * The element's own rectangle, and the screen it is in. The menu carries its own anchor rather + * than borrowing the outline's: a hover is dropped by anything that invalidates the tree, and a + * menu the user explicitly asked for must not vanish with it. + */ + bounds: ScreenBounds; + screen: ScreenSize; + }; + run?: TestRunView; + error: string; +}; + +/** + * How long a passing tick stays before it starts to go, and how long going takes. + * + * The ticks go and the verdict stays. Once a run has passed, a column of green beside every line + * says nothing the chip above the preview does not, and it buries the code the user came back to + * read; the chip and the duration are the part worth keeping on screen. A failing mark is the + * exception in both directions: it is the one thing that says *where*, so it never fades. + */ +const HOLD_MS = 2_500; +const FADE_MS = 600; + +const DEFAULT_TITLE = "a recorded flow"; +/** + * The runner reported no elements at all, so there is nothing a recorded step could name. It is + * the app, not the read path: the snapshot is rooted at the application and never carries the + * simulator's own chrome. + */ +const NO_TREE = "The app is not reporting any elements, so a recorded step would name nothing the test runner can find. Bring the app to the foreground, then record again."; +/** + * How long a tree read stays usable. Any input invalidates it outright; this only bounds the case + * where the app changed the screen on its own (an animation finishing, data arriving) without the + * user touching it. + */ +const TREE_TTL_MS = 1_500; + +/** + * Records taps on the mirror as deterministic test steps. + * + * One recording at a time, across the whole panel: a test is a single flow on a single device, and + * two concurrent recordings would interleave steps from two screens into one file. That is why + * this is a module singleton rather than a per-stage store. + */ +/** A fresh flow identity. Not a step id and never emitted: it only ever leaves the panel's memory. */ +function newFlowId(): string { + return crypto.randomUUID(); +} + +export class TestRecorder { + private readonly listeners = new Set<() => void>(); + private reader?: RecorderReader; + private components: ScreenComponent[] = []; + private componentsReadAt = 0; + private pending?: Promise<ScreenComponent[]>; + /** Only the newest hover may write the outline; earlier reads that land late are dropped. */ + private hoverToken = 0; + /** Where the pointer is, how far out of it the user has walked, and what it was last over. */ + private hoverPoint?: ScreenPoint; + private hoverLevel = 0; + private hoverInnermost?: string; + /** Where the open element menu resolved from, so a chosen row records the same element. */ + private menuPoint?: ScreenPoint; + /** A menu is being resolved. Set before the tree read, because the race is during it. */ + private menuOpening = false; + /** + * The menu's own concurrency, separate from the hover's. + * + * A hover that lands late must not draw over the element the user right-clicked, and — the bug + * this separation fixes — anything that bumps the *hover* token while the menu is reading must + * not silently drop the menu. A pointer move, an input that invalidates the tree, a frame: the + * recorder has several reasons to invalidate a hover and none of them are a reason to cancel a + * menu the user asked for. Only another `openMenu` or a `closeMenu` cancels this one. + */ + private menuToken = 0; + /** The centre of the element the last step named, which is where its action lands. */ + private lastTarget?: ScreenPoint; + /** That element's rectangle, which is the box a scoped swipe is drawn inside. */ + private lastBounds?: ScreenBounds; + private snapshot: TestRecorderSnapshot = { + recordingDeviceId: "", recordingPlatform: "", agentDeviceId: "", steps: [], title: DEFAULT_TITLE, revision: 0, flowId: newFlowId(), resolving: false, preparing: false, error: "", + }; + + private fadeTimer?: ReturnType<typeof setTimeout>; + + private treeSource?: RecorderTreeSource; + private readonly readers = new Map<string, RecorderReader>(); + + subscribe = (listener: () => void) => { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }; + getSnapshot = () => this.snapshot; + + /** The Testing panel supplies the tree source; see `RecorderTreeSource`. */ + useTreeSource(source?: RecorderTreeSource) { + if (this.treeSource === source) return; + this.treeSource?.release(); + this.treeSource = source; + this.invalidate(); + } + + /** What a panel's `readComponents` forwards to. Without a source there is nothing to record from. */ + readTree(deviceId: string, platform: DevicePlatform): Promise<{ tree: ScreenComponent[]; screen?: ScreenSize }> { + return this.treeSource ? this.treeSource.read(deviceId, platform) : Promise.resolve({ tree: [] }); + } + + releaseTree() { this.treeSource?.release(); } + + private update(fields: Partial<TestRecorderSnapshot>) { + // Anything that changes the emitted source is a new revision; the controls are not. + const changed = fields.steps !== undefined || fields.sourceOverride !== undefined || (fields.title !== undefined && fields.title !== this.snapshot.title); + // A run's marks are tied to lines of the file as it was emitted, so the first edit to the + // steps — recorded, deleted, dragged, duplicated, re-pointed — moves the lines out from under + // them. They go; the verdict and the duration stay, because those are about a run that did + // happen and the user is often reading them while writing the next step. Clearing the whole + // run here would also make `start()`'s own clear indistinguishable from an edit. + const run = fields.run === undefined && fields.steps !== undefined && this.snapshot.run + && this.snapshot.run.state !== "running" && Object.keys(this.snapshot.run.lines).length > 0 + ? { run: { ...this.snapshot.run, lines: {}, stale: true as const } } + : {}; + this.snapshot = { ...this.snapshot, ...fields, ...run, ...(changed ? { revision: this.snapshot.revision + 1 } : {}) }; + for (const listener of this.listeners) listener(); + } + + /** The simulator panel hands the recorder its way of reading the device it shows. */ + attach(deviceId: string, platform: DevicePlatform, reader: RecorderReader) { + const key = this.deviceKey(deviceId, platform); + this.readers.set(key, reader); + if (this.snapshot.recordingDeviceId && !this.isRecordingDevice(deviceId, platform)) return; + // The panel re-attaches whenever it refreshes its device list, which is every few seconds. + // Invalidating on each of those threw the cached tree away — and with it, through + // `clearHover`, the outline and any open element menu, about a second after they appeared. + // Only a reader that actually changed is a reason to read the screen again. + if (this.reader === reader) return; + // A reader being replaced held a session on whatever it was reading; hand that back first. + if (this.reader) this.reader.release?.(); + this.reader = reader; + this.invalidate(); + } + + /** The panel's device went away, so nothing can be recorded from it any more. */ + detach(deviceId: string, platform: DevicePlatform) { + const key = this.deviceKey(deviceId, platform); + const detached = this.readers.get(key); + this.readers.delete(key); + if (this.snapshot.recordingDeviceId && !this.isRecordingDevice(deviceId, platform)) { + return; + } + if (detached !== this.reader) { detached?.release?.(); return; } + const reader = this.reader; + this.reader = undefined; + this.invalidate(); + // `stop` releases through the reader, which is why the reader outlives this assignment until + // after it; with nothing recording there is still a session to give back. + if (this.snapshot.recordingDeviceId) { this.reader = reader; this.stop(); this.reader = undefined; } + else reader?.release?.(); + } + + private isRecordingDevice(deviceId: string, platform: DevicePlatform) { + return this.snapshot.recordingDeviceId === deviceId && this.snapshot.recordingPlatform === platform; + } + + private deviceKey(deviceId: string, platform: DevicePlatform) { return `${platform}\u0000${deviceId}`; } + + isRecording(deviceId?: string, platform?: DevicePlatform) { + return Boolean(deviceId) && platform !== undefined && this.isRecordingDevice(deviceId!, platform); + } + + /** + * Whether an agent is recording, so the mirror's own input is locked. + * + * Two drivers on one screen is solved by locking, not by hoping: a tap the user lands between + * the agent's read and its step resolves the step against a screen that has already moved. + */ + isAgentRecording(deviceId?: string) { + return this.snapshot.agentDeviceId !== "" && (deviceId === undefined || this.snapshot.agentDeviceId === deviceId); + } + + /** + * Takes the steps of an agent recording the server holds, as they arrive. + * + * `recordingDeviceId` is set for the agent's device, so the step list, the preview, editing, + * duplication and Save all work exactly as they do for a recording the user tapped out: an + * agent-recorded flow is an ordinary unsaved recording that happened to be written by something + * else. The steps replace rather than append — the server holds the whole flow, and reconciling + * two lists would be a second source of truth. + */ + adoptAgentRecording(deviceId: string, platform: DevicePlatform, steps: readonly RecordedStep[], title?: string) { + if (!deviceId) return; + this.reader = this.readers.get(this.deviceKey(deviceId, platform)) ?? this.reader; + const startedAt = performance.now(); + const added = Math.max(0, steps.length - this.snapshot.steps.length); + const began = this.snapshot.agentDeviceId !== deviceId; + if (began) { + this.clearHover(); + this.clearRun(); + countUiEvent("ui.testing.agent_session_start"); + } + const sameSteps = steps.length === this.snapshot.steps.length + && steps.every((step, index) => step.id === this.snapshot.steps[index]?.id); + if (!began && sameSteps && (title === undefined || title === this.snapshot.title)) return; + this.update({ + agentDeviceId: deviceId, recordingDeviceId: deviceId, recordingPlatform: platform, error: "", menu: undefined, + steps: [...steps], sourceOverride: undefined, + ...(began ? { flowId: newFlowId() } : {}), + ...(title ? { title } : {}), + }); + // Counted as steps, never as what they say: an element's name and the goal sentence are the + // user's own content and are not sent. See docs/telemetry.md. + for (let index = 0; index < added; index += 1) countUiEvent("ui.testing.agent_step_adopted"); + recordUiTiming("ui.testing.agent_adopt", performance.now() - startedAt); + } + + /** + * The agent is done. The steps stay, dirty, and the flow is an ordinary unsaved recording from + * here — so this stops the recorder exactly as the Stop button does, device release included. + * Leaving it armed would turn the user's next tap on the mirror into a step the agent never + * recorded, and leaving the session open is the leak `TestingDevices.release` exists to stop. + */ + endAgentRecording(reason: "finished" | "aborted") { + if (!this.snapshot.agentDeviceId) return; + this.update({ agentDeviceId: "" }); + this.stop(); + if (reason === "aborted") countUiEvent("ui.testing.agent_session_abort"); + } + + get source() { return this.snapshot.sourceOverride ?? this.emitted.source; } + + /** Switching to code editing makes source the authority for this flow. */ + editSource(source: string) { + if (source === this.source || this.snapshot.agentDeviceId) return; + this.stop(); + this.clearRun(); + this.update({ sourceOverride: source }); + countUiEvent("ui.testing.source_edit"); + } + + /** + * Begins recording. A recording that was already saved starts a new flow rather than appending + * to the file just written — the saved steps are still on screen, and a tap landing on the end + * of a finished test is never what was meant. + */ + start(deviceId: string, platform: DevicePlatform, keepSteps = false) { + const reader = this.readers.get(this.deviceKey(deviceId, platform)); + if (!deviceId || !reader || this.snapshot.agentDeviceId || (keepSteps && this.snapshot.sourceOverride !== undefined)) return; + if (this.reader !== reader) { + this.reader?.release?.(); + this.reader = reader; + this.invalidate(); + } + const had = this.snapshot.steps.length > 0; + this.invalidate(); + this.clearHover(); + // A run's marks are a verdict on the file as it was. Resuming adds steps that verdict never + // covered and starting again replaces the file entirely, so either way the overlay would be + // marking lines of something else — and a passing run's marks are mid-fade besides. + this.clearRun(); + // A flow that is not being resumed is a different recording, however much of the old one is + // still on screen a moment before. Anything holding on to the last flow — the panel's kept + // draft above all — has to be told, or the next thing written lands on the wrong one. + this.update({ recordingDeviceId: deviceId, recordingPlatform: platform, error: "", menu: undefined, preparing: true, ...(keepSteps ? {} : { steps: [], sourceOverride: undefined, flowId: newFlowId() }) }); + countUiEvent("ui.testing.record_start"); + // The first tree read is what makes an element selectable, and it is the slow part: the + // session attaches to the app and the tree is projected before anything can be named. Warming + // it here rather than on the first hover means the wait happens once, while the overlay is + // saying so, instead of silently under the user's first few clicks. + void this.prepare(deviceId, platform); + // Which way the user answered, counted only where there was a question to answer. It says + // whether the choice is carrying its weight; nothing about the flow itself is sent. + if (had) countUiEvent(keepSteps ? "ui.testing.record_resume" : "ui.testing.record_new"); + } + + /** + * Reads the screen once so the first hover or right-click is answered locally. + * + * Measured as its own timing: how long Record is unusable for is a number the panel is judged on, + * and it is not the same as `ui.testing.tree_read`, which also covers the warm reads during a + * recording. A read that fails leaves `preparing` false — the overlay must never outlive the + * thing it is explaining — and the next hover retries as it always did. + */ + private async prepare(deviceId: string, platform: DevicePlatform): Promise<void> { + const startedAt = performance.now(); + try { + await this.currentComponents(); + } catch { + // Silent: a failed warm read is not an error the user asked for. The first hover says so. + } finally { + recordUiTiming("ui.testing.record_prepare", performance.now() - startedAt, platform); + // Only the recording this prepared for: Stop, or a second Record, owns the flag after that. + if (this.snapshot.preparing && this.isRecordingDevice(deviceId, platform)) this.update({ preparing: false }); + } + } + + stop() { + if (!this.snapshot.recordingDeviceId) return; + this.clearHover(); + this.update({ recordingDeviceId: "", recordingPlatform: "", agentDeviceId: "", resolving: false, preparing: false, menu: undefined }); + // The steps stay; the device does not. See `RecorderReader.release`. + this.reader?.release?.(); + countUiEvent("ui.testing.record_stop"); + } + + /** + * Previews what a tap at this point would record, so the mirror can outline it — the same + * resolution the tap itself runs, against the same cached tree, so what is outlined is what is + * recorded. Call with nothing when the pointer leaves. + * + * Hovering reads the tree when it is stale, exactly as a tap does; once warm, every move after + * that is answered locally. + */ + async hover(point?: ScreenPoint) { + // An open menu owns the outline: it is about one element, and following the pointer would move + // the menu off the element whose steps it is listing. `menuOpening` covers the read the menu is + // waiting on — a browser sends a pointermove after the right-click, and a hover landing in that + // window would invalidate the menu's own token and silently drop it. + if (this.snapshot.menu || this.menuOpening) return; + if (!this.snapshot.recordingDeviceId || !point) { this.clearHover(); return; } + const token = ++this.hoverToken; + let components: ScreenComponent[]; + try { components = await this.currentComponents(); } + catch { return; } + // The pointer moved on, or recording stopped, while the tree was being read. + if (token !== this.hoverToken || !this.snapshot.recordingDeviceId) return; + this.hoverPoint = point; + // Walking out is anchored to one element: pointing at a different one starts again from it. + const innermost = hitLadder(components, point, this.reader?.screen())[0]?.nodeId; + if (innermost !== this.hoverInnermost) { this.hoverInnermost = innermost; this.hoverLevel = 0; } + this.showHover(components); + } + + /** + * Walks the outline out to the enclosing element and back in, for when the element under the + * pointer is not the one the test should name — a label inside the row, the row inside the card. + * It re-resolves from the cached tree, so it costs nothing and stays in step with the tap. + */ + walkHover(outwards: boolean) { + if (!this.snapshot.recordingDeviceId || !this.hoverPoint) return; + const next = this.hoverLevel + (outwards ? 1 : -1); + const levels = this.snapshot.hover?.levels ?? 1; + if (next < 0 || next > levels - 1) return; + this.hoverLevel = next; + this.showHover(this.components); + // An open menu is about the element the outline shows, so walking the outline re-asks what can + // be done with the element it has walked to. + if (this.snapshot.menu) this.showMenu(this.components); + countUiEvent("ui.testing.hover_walk"); + } + + private clearHover() { + this.hoverPoint = undefined; + this.menuPoint = undefined; + this.menuToken++; + this.hoverInnermost = undefined; + this.hoverLevel = 0; + // A menu resolved from a tree that is going is a menu about a screen that has gone. + if (this.snapshot.hover || this.snapshot.menu) this.update({ hover: undefined, menu: undefined }); + } + + private showHover(components: ScreenComponent[]) { + const point = this.hoverPoint; + const screen = this.reader?.screen(); + if (!point || !screen) return; + if (treeHealth(components) === "no-accessibility") { + if (this.snapshot.error !== NO_TREE) this.update({ error: NO_TREE }); + if (this.snapshot.hover) this.update({ hover: undefined }); + return; + } + const { selector, ambiguous, component, levels } = resolveSelector(components, point, screen, this.hoverLevel); + // Nothing under the pointer is still worth drawing: silence reads as a broken recorder, and + // the mark is the honest answer — this is the coordinate a tap here would record. + const bounds = component?.bounds ?? { x: point.x - 12, y: point.y - 12, width: 24, height: 24 }; + const label = stepLabel("tap", selector); + const hover = { bounds, label, screen, ambiguous, level: this.hoverLevel, levels }; + const previous = this.snapshot.hover; + if (previous && previous.label === label && previous.ambiguous === ambiguous && previous.level === hover.level + && previous.levels === levels && previous.bounds.x === bounds.x && previous.bounds.y === bounds.y + && previous.bounds.width === bounds.width && previous.bounds.height === bounds.height) return; + this.update({ hover }); + } + + /** + * Puts a kept recording back in the panel, without arming the recorder. + * + * A recorded flow comes back as steps; a code-edited flow also restores its source override. + * Nothing starts recording — the + * user reopened a flow to look at it, and a tap on the mirror a moment later must not append to + * it — and any run marks go, because they were a verdict on whatever was in the list before. + */ + restore(steps: readonly RecordedStep[], title?: string, source?: string) { + if (this.snapshot.agentDeviceId) return; + this.clearHover(); + this.clearRun(); + this.update({ + recordingDeviceId: "", recordingPlatform: "", resolving: false, preparing: false, error: "", menu: undefined, + steps: [...steps], sourceOverride: source, flowId: newFlowId(), ...(title ? { title } : {}), + }); + countUiEvent("ui.testing.draft_restore"); + } + + setTitle(title: string) { this.update({ title: title.trim() || DEFAULT_TITLE }); } + removeStep(id: string) { this.update({ steps: this.snapshot.steps.filter(step => step.id !== id) }); } + clearSteps() { this.update({ steps: [], sourceOverride: undefined, flowId: newFlowId() }); } + + /** + * Appends a copy of one step at the end of the list. + * + * At the end rather than beside the original: a recorded flow is read top to bottom, and the + * step being copied is almost always one the user wants to repeat *after* what follows it — + * tapping the same disclosure again to collapse it, asserting the same row twice. Dragging it + * into place is the other half of this, and it is cheaper than deciding for them. + */ + duplicateStep(id: string) { + const step = this.snapshot.steps.find(entry => entry.id === id); + if (!step) return; + this.update({ steps: [...this.snapshot.steps, { ...step, id: crypto.randomUUID() }] }); + countUiEvent("ui.testing.step_duplicate"); + } + + /** + * Swaps what a step does while keeping the element it named. + * + * The selector is the expensive half of a step — it was resolved against a screen that has since + * moved on — so changing the command must not cost the user that screen. Nothing is sent to the + * device: the flow has already run past this point, and re-taking a step out of order would + * leave the app somewhere the rest of the list does not describe. + */ + changeStepCommand(id: string, kind: PinnedStepKind, value?: string, timeoutMs?: number) { + const steps: RecordedStep[] = this.snapshot.steps.map(step => { + // An agent step has no selector to keep, which is the whole reason this is offered: the + // mirror's Agentic branch records one, and nothing converts between the two shapes. + if (step.id !== id || isAgentStep(step)) return step; + // A kind that carries no text drops the old one rather than keeping a value nothing emits. + const next = stepNeedsValue(kind) ? value ?? step.value ?? "" : undefined; + // Both of the optional halves go, so a step never keeps the other command's leftovers: no + // text on a kind that emits none, no direction on a kind that is not a swipe. + const { value: _dropped, direction: _wasSwipe, timeoutMs: _wasTimeout, ...rest } = step; + return { + ...rest, + kind, + label: stepLabel(kind, step.selector, next), + ...(kind === "swipe" ? { direction: swipeDirection(value) } : next === undefined ? {} : { value: next }), + ...(stepIsAssertion(kind) && timeoutMs !== undefined ? { timeoutMs } : {}), + }; + }); + if (steps.every((step, index) => step === this.snapshot.steps[index])) return; + this.update({ steps }); + countUiEvent(stepIsAssertion(kind) ? "ui.testing.step_recommand_assert" : "ui.testing.step_recommand_action"); + } + + /** + * Moves a step to another position in the list, which is the order the file is emitted in. + * + * Everything derived from position — the numbering, the preview, the lines a run marks — is + * recomputed from `steps`, so the move is the whole change. + */ + moveStep(id: string, toIndex: number) { + const from = this.snapshot.steps.findIndex(step => step.id === id); + if (from < 0) return; + const to = Math.max(0, Math.min(this.snapshot.steps.length - 1, toIndex)); + if (to === from) return; + const steps = [...this.snapshot.steps]; + const [moved] = steps.splice(from, 1); + steps.splice(to, 0, moved!); + this.update({ steps }); + countUiEvent("ui.testing.step_reorder"); + } + dismissError() { this.update({ error: "" }); } + + /** + * Marks the cached tree stale. Every input changes the screen, so the panel calls this after + * each one rather than letting a tap be named from the screen it already left. + */ + invalidate() { + this.hoverToken++; + this.clearHover(); + this.components = []; + this.componentsReadAt = 0; + this.pending = undefined; + } + + private async currentComponents(): Promise<ScreenComponent[]> { + const fresh = this.componentsReadAt > 0 && performance.now() - this.componentsReadAt < TREE_TTL_MS; + if (fresh && this.components.length) return this.components; + if (!this.reader) return []; + this.pending ??= (async () => { + const startedAt = performance.now(); + try { + const components = await this.reader!.readComponents(); + this.components = components; + this.componentsReadAt = performance.now(); + return components; + } finally { + recordUiTiming("ui.testing.tree_read", performance.now() - startedAt, this.snapshot.recordingPlatform || undefined); + this.pending = undefined; + } + })(); + return this.pending; + } + + /** + * Appends one step for the element the outline is on. Shared by the tap path and the element + * menu, so a step means the same thing however it was asked for. + * + * Returns whether a step was added; a refusal has already said why. + */ + private addStep(components: ScreenComponent[], point: ScreenPoint, kind: PinnedStepKind, value?: string, timeoutMs?: number): boolean { + const screen = this.reader?.screen(); + // The outline the user is looking at is the step they mean, walked-out level and all. + const { selector, ambiguous, component: named } = resolveSelector(components, point, screen, this.hoverLevel); + const target = named && { ...(named.role === undefined ? {} : { role: named.role }), ...(named.placeholder === undefined ? {} : { placeholder: named.placeholder }), ...(named.label === undefined ? {} : { label: named.label }), ...(named.text === undefined ? {} : { text: named.text }), ...(named.value === undefined ? {} : { value: named.value }) }; + // A coordinate has no element, so most commands would emit nothing — but a tap lands on it and + // `screen.swipe` is about the viewport rather than about anything in it, so both go through. + if (selector.kind === "point" && kind !== "tap" && kind !== "swipe") { + // A coordinate is not a thing to be visible, to hold text or to be scrolled to, so the step + // would emit nothing. Say so instead of adding a step whose whole body is two comments. + countUiEvent("ui.testing.step_unresolved"); + this.update({ error: "Nothing here can be named, so there is nothing to act on but the coordinate. Scroll the mirror to pick the element around it, or add a testID." }); + return false; + } + const step: PinnedStep = { + id: crypto.randomUUID(), + kind, + selector, + label: stepLabel(kind, selector, value), + ...(kind === "swipe" ? { direction: swipeDirection(value) } : value === undefined ? {} : { value }), + ...(stepIsAssertion(kind) && timeoutMs !== undefined ? { timeoutMs } : {}), + ...(ambiguous ? { ambiguous } : {}), + // Only a coordinate tap needs one: every locator action waits for its own element. + ...(selector.kind === "point" ? { anchor: anchorSelector(components, point, screen) } : {}), + // Kept so the step list can offer the same commands later; see `RecordedStep.target`. + ...(target ? { target } : {}), + }; + // Where the runner's own locator action would land, for `chooseOffer` to act on. + const owner = resolveSelector(components, point, screen, this.hoverLevel).component; + // A locator swipe is scoped to the element, a viewport swipe to the screen; `undefined` here + // is what makes `chooseOffer` fall back to the whole screen. + this.lastBounds = selector.kind === "point" ? undefined : owner?.bounds; + this.lastTarget = owner + ? { x: owner.bounds.x + owner.bounds.width / 2, y: owner.bounds.y + owner.bounds.height / 2 } + : point; + this.update({ steps: [...this.snapshot.steps, step] }); + countUiEvent(kind === "tap" ? "ui.testing.step_tap" : stepIsAssertion(kind) ? "ui.testing.step_assert" : "ui.testing.step_action"); + if (selector.kind === "point") countUiEvent("ui.testing.step_unresolved"); + return true; + } + + /** + * Opens the element menu over the mirror: what the runner can do with the element the outline is + * currently on, each row carrying the action and emitted code the step would add. + * + * It resolves from the same cached tree and the same walked-out level as the outline, so the + * menu is about the element the user can see outlined — including one they walked out to with + * the wheel — rather than whatever happens to be innermost under the pointer. + */ + async openMenu(point: ScreenPoint) { + if (!this.snapshot.recordingDeviceId) return; + const token = ++this.menuToken; + let components: ScreenComponent[]; + this.menuOpening = true; + this.update({ resolving: true }); + try { components = await this.currentComponents(); } + catch (error) { this.update({ error: error instanceof Error ? error.message : String(error) }); return; } + finally { this.menuOpening = false; this.update({ resolving: false }); } + if (token !== this.menuToken || !this.snapshot.recordingDeviceId) return; + // The menu owns the outline from here, so a hover still in flight cannot move it. + this.hoverToken++; + if (treeHealth(components) === "no-accessibility") { this.update({ error: NO_TREE }); return; } + // The right-click is also a hover: it is where the user is pointing, and the outline follows it. + this.hoverPoint = point; + const innermost = hitLadder(components, point, this.reader?.screen())[0]?.nodeId; + if (innermost !== this.hoverInnermost) { this.hoverInnermost = innermost; this.hoverLevel = 0; } + this.showHover(components); + this.menuPoint = point; + this.showMenu(components); + countUiEvent("ui.testing.menu_open"); + } + + /** The menu for the element the outline currently shows, resolved from the cached tree. */ + private showMenu(components: ScreenComponent[]) { + const point = this.menuPoint; + const screen = this.reader?.screen(); + if (!point || !screen) return; + const { selector, ambiguous, component, matches } = resolveSelector(components, point, screen, this.hoverLevel); + // Nothing resolved is still somewhere: the menu marks the coordinate a tap here would record. + const bounds = component?.bounds ?? { x: point.x - 12, y: point.y - 12, width: 24, height: 24 }; + this.update({ menu: { + label: stepLabel("tap", selector), ambiguous, matches, + offers: stepOffers(selector, component), + commands: allStepOffers(selector, component), + ...(component ? { target: component } : {}), + bounds, screen, + } }); + } + + closeMenu() { + this.menuOpening = false; + this.menuToken++; + if (this.snapshot.menu) this.update({ menu: undefined }); + } + + /** + * Records an `agent.act` or `agent.assert` step from the sentence the user wrote. + * + * It does not go through `addStep`, and that is the point: there is no selector to resolve, so + * the tree is never read and the coordinate refusal never applies — an element nothing can name + * is exactly where describing it is the only way to write the step. The element the user + * right-clicked only ever prefilled the sentence; it is not carried onto the step and never + * reaches the file, because the runner resolves the instruction against the screen it finds. + * + * Nothing is sent to the device either. Taking an agent step here would mean running a model + * against the live app mid-recording, which is a run, not a recording. + */ + recordAgentStep(chosen: + | { kind: "agentAct"; instruction: string; params?: AgentParams } + | { kind: "agentAssert" | "agentWaitFor"; instruction: string; vision?: VisionMode }) { + const instruction = chosen.instruction.trim(); + this.update({ menu: undefined }); + if (!instruction || !this.snapshot.recordingDeviceId) return; + const kind: AgentStepKind = chosen.kind; + // The optional halves — `params` on an act, `vision` on an assert — are carried exactly as the + // form left them, and the form leaves a default off rather than writing it out: no params from + // rows the user never filled in, no `vision: false` where `false` is what the runner does. + const step: AgentStep = { ...chosen, instruction, id: crypto.randomUUID(), label: agentStepLabel(kind, instruction) }; + this.update({ steps: [...this.snapshot.steps, step] }); + countUiEvent(kind === "agentAct" ? "ui.testing.step_agent_act" + : kind === "agentWaitFor" ? "ui.testing.step_agent_wait" : "ui.testing.step_agent_assert"); + // Counted apart from the step itself: whether params are reached for at all is the question, + // and the count is of the rows, never of a key or a value. + if (chosen.kind === "agentAct" && chosen.params) setUiGauge("ui.testing.agent_params", Object.keys(chosen.params).length); + } + + /** + * Records the step a menu row stands for. Nothing is sent to the device: the menu is how a user + * records an assertion, a scroll or a typed value without the tap that would change the screen + * out from under it. + */ + async chooseOffer(kind: PinnedStepKind, value?: string, timeoutMs?: number) { + const point = this.menuPoint; + this.update({ menu: undefined }); + if (!point || !this.snapshot.recordingDeviceId) return; + this.update({ resolving: true }); + try { + const components = await this.currentComponents(); + if (treeHealth(components) === "no-accessibility") { this.update({ error: NO_TREE }); return; } + const added = this.addStep(components, point, kind, value, timeoutMs); + if (!added) return; + // The step is taken as well as recorded, so the app is on the screen the next step will be + // resolved against. `where` is the element's own centre, which is where the runner's locator + // action lands — not the pixel the user happened to right-click. + const effect = stepEffect(kind); + const where = this.lastTarget ?? point; + if (effect === "tap") this.reader?.perform?.({ kind: "tap", point: where }); + // The panel's input path has one tap in it, so a double tap is sent as two. It is the screen + // that has to keep up with the recorded flow, and an app that treats the pair as two taps is + // one the emitted `doubleTap` would behave differently on — the step is right either way. + else if (effect === "doubleTap") { + this.reader?.perform?.({ kind: "tap", point: where }); + this.reader?.perform?.({ kind: "tap", point: where }); + } + else if (effect === "swipe") { + const span = this.lastBounds; + this.reader?.perform?.({ kind: "swipe", ...swipePath(span ?? { x: 0, y: 0, ...(this.reader?.screen() ?? { width: 0, height: 0 }) }, swipeDirection(value)) }); + } + else if (effect === "type") { + this.reader?.perform?.({ kind: "tap", point: where }); + this.reader?.perform?.({ kind: "type", text: value ?? "" }); + } + // Anything that touched the app leaves a screen the cached tree no longer describes. + if (effect !== "none") this.invalidate(); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + this.update({ resolving: false }); + } + } + + /** The emitted file and where each step's lines are in it. See `emitE2eTestLines`. */ + get emitted() { return emitE2eTestLines(this.snapshot.steps, { title: this.snapshot.title }); } + + /** + * The same test cut off after one step: what Play until here runs. + * + * It is a re-emit of the first *n* steps rather than a slice of the source text, so the file it + * produces is a whole test — the header, the steps up to and including this one, and the closing + * brace. Every line it keeps stays at the line number it has in the full file: the header is a + * fixed seven lines whether or not an agent step is left in it, and the only thing removed is + * the tail. That is what lets the runner's report mark the preview's own lines. + * + * `untilLine` is the last line of the step that was asked for, which is where the run stops. + * + * `OPEN_STEP` is the flow with no steps at all: the app opened and nothing done to it. It is a + * real thing to ask for — the app launching is where a flow most often breaks — and it is the + * one line of the header the runner reports, so it marks like any other. + */ + emittedThrough(stepId: string): { source: string; untilLine: number } | undefined { + if (this.snapshot.sourceOverride !== undefined) return; + const steps = this.snapshot.steps; + if (stepId === OPEN_STEP) { + const emitted = emitE2eTestLines([], { title: this.snapshot.title }); + const line = emitted.calls[0]; + return line === undefined ? undefined : { source: emitted.source, untilLine: line }; + } + const index = steps.findIndex(step => step.id === stepId); + if (index < 0) return; + const span = this.emitted.spans[index]; + if (!span) return; + return { source: emitE2eTest(steps.slice(0, index + 1), { title: this.snapshot.title }), untilLine: span.to }; + } + + /** + * A run has started. Every call line goes to `running` together, because that is as much as the + * command line honestly tells us while the runner works — see `packages/core/src/testing-run.ts` + * for why there is no per-step stream to follow. + */ + beginRun(untilLine?: number) { + // A fade left over from the previous run would take this run's marks with it. + this.stopFade(); + this.update({ run: { state: "running", lines: {}, ...(untilLine === undefined ? {} : { untilLine }) } }); + } + + /** + * One batch of the runner's live progress, mapped onto lines. + * + * A `step` event has no source position — only the finished report does — but it has the step's + * index within its test, and the runner reports one step per awaited call in order, so the index + * is the position in the supplied call lines. Recorded flows default to `EmittedTest.calls`; + * saved project tests supply the awaited lines parsed from their source. A step that has started + * is running; one that has ended takes its own status. The report reconciles all of it at the end. + */ + applyRunProgress(entries: readonly { stepIndex: number; phase: "start" | "end"; status?: string; error?: string }[], calls = this.emitted.calls) { + const run = this.snapshot.run; + if (!run || run.state !== "running") return; + const lines: Record<number, RunLineMark> = {}; + let error: string | undefined; + for (const entry of entries) { + const line = calls[entry.stepIndex]; + if (line === undefined) continue; + if (entry.phase === "start") { lines[line] = "running"; continue; } + const mark: RunLineMark = entry.status === "failed" || entry.status === "timedOut" ? "failed" : "passed"; + lines[line] = mark; + if (mark === "failed" && !error) error = entry.error ?? `${entry.status} at line ${line}`; + } + const same = Object.keys(lines).length === Object.keys(run.lines).length + && Object.entries(lines).every(([line, mark]) => run.lines[Number(line)] === mark); + if (same && error === run.error) return; + this.update({ run: { ...run, lines, ...(error ? { error } : {}) } }); + } + + /** + * The run is over: each line takes the status the report gave it, and a line the runner never + * reached loses its mark rather than keeping a spinner that will never resolve. + * + * The verdict and the duration then stay above the preview until the next run begins, the steps + * are edited, or it is cleared — pass or fail, with no timeout. The marks in the preview are the + * half that still goes: the passing ticks fade shortly after the run, because a column of green + * repeats what the chip already says and buries the code underneath it, while the mark on a line + * that failed stays for good, since it is the only thing on screen that says where. See + * `fadePassedMarks`. + */ + finishRun(result: { status: string; steps: { line: number; status: string; error?: string }[]; output?: string; durationMs?: number }) { + this.stopFade(); + const lines: Record<number, RunLineMark> = {}; + let error: string | undefined; + for (const step of result.steps) { + const mark: RunLineMark = step.status === "passed" ? "passed" : step.status === "failed" || step.status === "timedOut" ? "failed" : "passed"; + // A retried line reports more than once; a failure is the one worth keeping. + if (lines[step.line] === "failed") continue; + lines[step.line] = mark; + if (mark === "failed" && !error) error = step.error ?? `${step.status} at line ${step.line}`; + } + const passed = result.status === "passed" && error === undefined; + // How far the run was asked to go is a fact about the run, so it outlives the run's own report: + // "Passed" on a flow that stopped three steps in has to keep saying so. + const untilLine = this.snapshot.run?.untilLine; + this.update({ run: { state: passed ? "passed" : "failed", lines, + ...(untilLine === undefined ? {} : { untilLine }), + ...(error ? { error } : {}), ...(result.output ? { output: result.output } : {}), + // Shown for a failure as well: how long a run took before it gave up is as much a part of + // reading it as how long a passing one took. + ...(result.durationMs === undefined ? {} : { durationMs: result.durationMs }) } }); + this.fadePassedMarks(); + } + + /** The run failed to start at all, or the user cleared it. */ + failRun(error: string) { + const untilLine = this.snapshot.run?.untilLine; + this.update({ run: { state: "failed", lines: {}, error, ...(untilLine === undefined ? {} : { untilLine }) } }); + } + clearRun() { + this.stopFade(); + if (this.snapshot.run) this.update({ run: undefined }); + } + + private stopFade() { + if (this.fadeTimer !== undefined) { clearTimeout(this.fadeTimer); this.fadeTimer = undefined; } + } + + /** + * Lets the passing ticks go, a while after the run, and leaves everything else alone. + * + * The run itself is untouched: the verdict and the duration stay above the preview until the + * next run, an edit or an explicit clear. Only the `passed` entries leave `lines`, so a mixed + * run keeps the mark on the line that failed while the ticks around it clear. + */ + private fadePassedMarks() { + this.stopFade(); + if (!Object.values(this.snapshot.run?.lines ?? {}).includes("passed")) return; + // A run whose marks are still worth fading: nothing has replaced it, and nothing has already + // dropped them — an edit to the steps takes them all at once and sets `stale`. + const fadable = () => { + const run = this.snapshot.run; + return run && run.state !== "running" && !run.stale + && Object.values(run.lines).includes("passed") ? run : undefined; + }; + this.fadeTimer = setTimeout(() => { + this.fadeTimer = undefined; + const run = fadable(); + if (!run) return; + this.update({ run: { ...run, fading: true } }); + this.fadeTimer = setTimeout(() => { + this.fadeTimer = undefined; + const current = this.snapshot.run; + if (!current) return; + const { fading: _gone, ...rest } = current; + this.update({ run: { ...rest, + lines: Object.fromEntries(Object.entries(current.lines).filter(([, mark]) => mark !== "passed")) } }); + }, FADE_MS); + }, HOLD_MS); + } +} + +/** + * How long a run took, the way the `e2e` CLI writes it. + * + * Seconds to two decimals once there is a second to show, and whole milliseconds below that: a run + * that took 840ms reads as a number, where `0.84s` reads as a rounding of something longer. + */ +export function formatRunDuration(durationMs: number): string { + if (!Number.isFinite(durationMs) || durationMs < 0) return ""; + return durationMs < 1000 ? `${Math.round(durationMs)}ms` : `${(durationMs / 1000).toFixed(2)}s`; +} + +/** The one recorder. See the class comment for why there is only one. */ +export const testRecorder = new TestRecorder(); diff --git a/packages/ui/src/test-source-suggestions.ts b/packages/ui/src/test-source-suggestions.ts new file mode 100644 index 00000000..7d8d710e --- /dev/null +++ b/packages/ui/src/test-source-suggestions.ts @@ -0,0 +1,111 @@ +/** Completion entries for the test API emitted by the recorder. No project or TypeScript indexing. */ +export type TestSourceSuggestion = { label: string; insert: string; cursorBack?: number; detail?: string }; +export type TestSourceCompletion = { from: number; to: number; items: TestSourceSuggestion[] }; + +const locatorMethods: TestSourceSuggestion[] = [ + { label: "getByTestId", insert: 'getByTestId("")', cursorBack: 2 }, + { label: "getByRole", insert: 'getByRole("", "")', cursorBack: 6 }, + { label: "getByText", insert: 'getByText("")', cursorBack: 2 }, + { label: "getByLabel", insert: 'getByLabel("")', cursorBack: 2 }, + { label: "getByPlaceholder", insert: 'getByPlaceholder("")', cursorBack: 2 }, +]; +const locatorActions: TestSourceSuggestion[] = [ + { label: "tap", insert: "tap()" }, + { label: "doubleTap", insert: "doubleTap()" }, + { label: "longPress", insert: "longPress()" }, + { label: "fill", insert: 'fill("")', cursorBack: 2 }, + { label: "nth", insert: "nth(0)", cursorBack: 2 }, +]; +const assertions: TestSourceSuggestion[] = [ + { label: "not", insert: "not." }, + { label: "toBeVisible", insert: "toBeVisible()", cursorBack: 1 }, + { label: "toHaveText", insert: 'toHaveText("")', cursorBack: 2 }, + { label: "toContainText", insert: 'toContainText("")', cursorBack: 2 }, + { label: "toHaveValue", insert: 'toHaveValue("")', cursorBack: 2 }, + { label: "toBeEnabled", insert: "toBeEnabled()", cursorBack: 1 }, + { label: "toBeDisabled", insert: "toBeDisabled()", cursorBack: 1 }, + { label: "toBeSelected", insert: "toBeSelected()", cursorBack: 1 }, +]; +const assertionStarts: TestSourceSuggestion[] = [ + { label: "expect visible", insert: 'expect(screen.getByTestId("")).toBeVisible()', cursorBack: 17, + detail: 'expect(screen.getByTestId("…")).toBeVisible()' }, + { label: "expect not visible", insert: 'expect(screen.getByTestId("")).not.toBeVisible()', cursorBack: 21, + detail: 'expect(screen.getByTestId("…")).not.toBeVisible()' }, + { label: "expect text", insert: 'expect(screen.getByTestId("")).toHaveText("")', cursorBack: 18, + detail: 'expect(screen.getByTestId("…")).toHaveText("…")' }, +]; +const agentMethods: TestSourceSuggestion[] = [ + { label: "act", insert: 'act("")', cursorBack: 2 }, + { label: "assert", insert: 'assert("")', cursorBack: 2 }, + { label: "assert with vision", insert: 'assert("", { vision: true })', cursorBack: 20, + detail: 'assert("…", { vision: true })' }, + { label: "waitFor", insert: 'waitFor("")', cursorBack: 2 }, +]; + +function matching(items: TestSourceSuggestion[], prefix: string, from: number, to: number): TestSourceCompletion | undefined { + const filtered = items.filter(item => item.label.toLowerCase().startsWith(prefix.toLowerCase()) && item.label !== prefix); + return filtered.length ? { from, to, items: filtered } : undefined; +} + +function inCommentOrString(line: string): boolean { + let quote = ""; + for (let index = 0; index < line.length; index++) { + const char = line[index]; + if (char === "\\" && quote) { index++; continue; } + if (quote) { if (char === quote) quote = ""; continue; } + if (char === "/" && line[index + 1] === "/") return true; + if (char === '"' || char === "'" || char === "`") quote = char; + } + return Boolean(quote); +} + +export function testSourceSuggestions(source: string, cursor: number): TestSourceCompletion | undefined { + const before = source.slice(0, cursor).split("\n").at(-1) ?? ""; + // Keep suggestions out of comments and quoted arguments. These are the recorder's single-line + // strings, not a full TypeScript parse; the editor still accepts arbitrary source. + if (inCommentOrString(before)) return; + const visionValue = before.match(/\bagent\.(?:assert|waitFor)\((?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'),\s*\{\s*vision:\s*(true|false|"only"|[A-Za-z"]*)$/); + if (visionValue) { + const prefix = visionValue[1]; + return matching([ + { label: "true", insert: "true" }, { label: "false", insert: "false" }, { label: '"only"', insert: '"only"' }, + ], prefix, cursor - prefix.length, cursor); + } + const agentOption = before.match(/\bagent\.(assert|waitFor|act)\((?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'),(\s*\{?\s*[A-Za-z]*)$/); + if (agentOption) { + const typed = agentOption[2].replace(/[^A-Za-z]/g, ""); + const options: TestSourceSuggestion[] = agentOption[1] === "act" + ? [{ label: "params", insert: " { params: {} }", cursorBack: 3, detail: "Pass named values to the agent" }] + : [ + { label: "vision: true", insert: " { vision: true }", detail: "Use the image when needed" }, + { label: 'vision: "only"', insert: ' { vision: "only" }', detail: "Use only the image" }, + { label: "vision: false", insert: " { vision: false }", detail: "Use the accessibility tree" }, + ]; + return matching(options, typed, cursor - agentOption[2].length, cursor); + } + const assertionCall = before.match(/\.(toBeVisible|toHaveText|toContainText|toHaveValue|toBeEnabled|toBeDisabled|toBeSelected)\(([^)]*)$/); + if (assertionCall) { + const [, method, args] = assertionCall; + const option = args.match(/(\s*\{?\s*[A-Za-z]*)$/); + const needsValue = method === "toHaveText" || method === "toContainText" || method === "toHaveValue"; + if (option && (!needsValue && args === option[1] || needsValue && /,\s*\{?\s*[A-Za-z]*$/.test(args))) { + const typed = option[1].replace(/[^A-Za-z]/g, ""); + return matching([{ label: "timeout", insert: "{ timeout: 30000 }", detail: "Wait up to 30000 ms" }], + typed, cursor - option[1].length, cursor); + } + } + const match = before.match(/\.([A-Za-z]*)$/); + if (!match) { + const beginning = before.match(/(?:^|\bawait\s+)([A-Za-z]+)$/); + if (!beginning) return; + return matching(assertionStarts, beginning[1], cursor - beginning[1].length, cursor); + } + const prefix = match[1]; + const owner = before.slice(0, match.index); + const entries = /(?:^|\W)screen$/.test(owner) ? locatorMethods + : /(?:^|\W)app$/.test(owner) ? [{ label: "open", insert: "open()" }] + : /(?:^|\W)agent$/.test(owner) ? agentMethods + : /expect\([^\n]*\)(?:\.not)?$/.test(owner) ? owner.endsWith(".not") ? assertions.filter(item => item.label !== "not") : assertions + : /screen\.(?:getBy\w+|tapAt)\(/.test(owner) ? locatorActions : []; + return matching(entries, prefix, cursor - prefix.length, cursor); +} diff --git a/packages/ui/src/testing-panel.ts b/packages/ui/src/testing-panel.ts new file mode 100644 index 00000000..a7a52b23 --- /dev/null +++ b/packages/ui/src/testing-panel.ts @@ -0,0 +1,1151 @@ +import type { HostBridge } from "./host-bridge.ts"; +import type { DeviceAppsStore } from "./device-apps.ts"; +import { captureUiError, countUiEvent, getUiTelemetryAttributes, recordUiTiming } from "./telemetry.ts"; +import { testRecorder } from "./test-recorder.ts"; +import type { RecorderTreeSource } from "./test-recorder.ts"; +import { isRecordableDevice, recordablePlatform } from "@mobile-dev/shared/panel-target.ts"; +import type { ScreenComponent } from "@mobile-dev/shared/screen-annotations.ts"; +import { screenSize } from "@mobile-dev/shared/screen-annotations.ts"; +import { testFileSlug } from "@mobile-dev/shared/test-recording.ts"; +import { storedTestCallLines, storedTestUntilSource } from "./stored-test-steps.ts"; +import type { RecordedStep } from "@mobile-dev/shared/test-recording.ts"; +import type { TestDraftSummary } from "@mobile-dev/shared/test-drafts.ts"; + +/** + * One test the project already holds, as the server reports it. + * + * The panel's list has two halves: what has not been stored yet (the drafts above) and what the + * project already holds. A saved recording moves from one to the other, which is the whole reason + * this exists — the draft is deleted on save, so before this the test left the list entirely. + */ +export type StoredTestSummary = { relativePath: string; name: string; updatedAt: string; sizeBytes: number }; + +/** + * An agent recording, as the server reports it. + * + * The recording itself lives in the server — see `packages/core/src/testing-agent.ts` for why it + * has to — and the panel polls for it, the same way it polls a run's progress and for the same + * reason: a tool call is one request and one response. + */ +export type TestingAgentView = { + active: boolean; + status: "idle" | "recording" | "finished" | "aborted"; + title: string; + goal: string; + steps: number; + note: string; + /** The unsaved recording a finished session was kept as; "" while it is still recording. */ + draftId: string; +}; + +/** 400 ms while an agent is recording, so a step lands in the panel as it is recorded. */ +const AGENT_POLL_ACTIVE_MS = 400; +/** + * And a slow tick otherwise, so a recording the agent starts is noticed even when the user is + * looking at another tool. It is what opens the Testing tab for them. + */ +const AGENT_POLL_IDLE_MS = 2_500; + +/** + * How long the panel waits after the last change before keeping the flow. + * + * Long enough that dragging a step or typing a title is one write rather than a dozen, short + * enough that switching tools a moment after a tap still finds the step. A write is a few + * kilobytes to a file in the plugin's own storage, so the cost of being wrong here is small in + * one direction and a lost recording in the other. + */ +const DRAFT_AUTOSAVE_MS = 700; + +/** One entry of the project selector. The server owns the list so a pick survives the panel document. */ +export interface TestingProjectOption { path: string; name: string } +/** A target as the project's e2e config declares it; a static read may not see every field. */ +export interface TestingTargetOption { name: string; platform?: string; bundleId?: string } + +export type TestingSnapshot = { + open: boolean; + available: boolean; + projectPath: string; + projects: TestingProjectOption[]; + targets: TestingTargetOption[]; + /** The target the panel records against; "" until a project with targets is loaded. */ + targetName: string; + configPath: string; + loadingProject: boolean; + pickingProject: boolean; + launching: boolean; + /** + * Whether this panel launched the app on the selected device. It is optimistic and panel-local: + * nothing polls the device for it, so an app launched elsewhere still reads as false here. It + * only drives the button's label, never anything the recorder depends on. + */ + appLaunched: boolean; + /** The file name under `tests/`, without the folder; "" means "derive it from the title". */ + fileName: string; + saving: boolean; + /** Set when the target file exists and the user has not yet agreed to replace it. */ + overwritePath: string; + /** The recorder revision that was written, so the panel can tell saved from dirty. */ + savedRevision: number; + savedPath: string; + /** A run is in flight. The recorder holds the per-line marks; this is just the button's state. */ + running: boolean; + /** Run asked to replace an existing file and is waiting on the answer. */ + pendingRun: boolean; + /** The agent recording the server holds, or null when no agent has ever recorded here. */ + agent: TestingAgentView | null; + /** + * The recordings that have not been written into a project yet, newest first. + * + * A flow lives in the panel document, which the host rebuilds whenever the user looks at + * another tool or another chat, so without this a recording they had not saved — or one their + * agent made while they were reading the chat — was gone when they came back. The panel keeps + * what it is holding as it changes and lists the rest, so one can be reopened or thrown away. + */ + drafts: TestDraftSummary[]; + /** The draft the flow on screen is being kept as; "" before the first autosave. */ + draftId: string; + /** A draft is being read back into the panel. */ + loadingDraft: boolean; + /** The `.e2e.ts` files the chosen project already holds, newest first. */ + tests: StoredTestSummary[]; + /** A project file shown as source, without importing it into the editable recorder. */ + selectedTestPath: string; + selectedTestSource: string; + selectedTestOriginalHash: string; + selectedTestStaged: boolean; + loadingTest: boolean; + /** A stored test is being deleted; its path, so only its own row shows the spinner. */ + deletingTest: string; + error: string | null; +}; + +function textError(result: Awaited<ReturnType<HostBridge["callServerTool"]>>, fallback: string): Error { + return new Error(result.content.filter(item => item.type === "text").map(item => item.text).join("\n") || fallback); +} +function object(value: unknown): Record<string, unknown> | null { + return value && typeof value === "object" ? value as Record<string, unknown> : null; +} +function text(value: unknown): string { return typeof value === "string" ? value : ""; } +function projectName(projectPath: string): string { return projectPath.split("/").filter(Boolean).at(-1) ?? projectPath; } + +/** + * The Testing panel: pick an e2e project, launch its app on the mirrored device, and (once the + * recorder lands) turn taps on the mirror into a deterministic test. + * + * This is the walking skeleton — project selection and launch only. Tap interception, selector + * resolution and codegen are specified in `docs/testing-panel.md` and are not implemented yet. + */ +export class TestingPanel { + private readonly app: HostBridge; + private readonly deviceApps: DeviceAppsStore; + private readonly listeners = new Set<() => void>(); + private disposed = false; + private projectRequest?: Promise<void>; + private restoreRequest?: Promise<void>; + /** Guards against a slow project load overwriting a newer selection. */ + private loadRevision = 0; + private testReadRevision = 0; + private stageWrite?: Promise<void>; + /** The project and target the installed tree source reads; "" when there is no source. */ + private treeSourceKey = ""; + private snapshot: TestingSnapshot = { + open: false, available: false, projectPath: "", projects: [], targets: [], targetName: "", configPath: "", + loadingProject: false, pickingProject: false, launching: false, appLaunched: false, + fileName: "", saving: false, overwritePath: "", savedRevision: -1, savedPath: "", running: false, pendingRun: false, + agent: null, drafts: [], draftId: "", loadingDraft: false, tests: [], selectedTestPath: "", selectedTestSource: "", selectedTestOriginalHash: "", selectedTestStaged: false, loadingTest: false, deletingTest: "", error: null, + }; + + /** The agent recording's revision the panel has already adopted; -1 before the first poll. */ + private agentRevision = -1; + private agentTimer?: ReturnType<typeof setTimeout>; + private agentPolling = false; + /** The last selection reported to the server, so the context tool is not called on every tick. */ + private reportedContext = ""; + private readonly stopWatchingDevices: () => void; + private readonly stopWatchingRecorder: () => void; + private draftTimer?: ReturnType<typeof setTimeout>; + /** The recorder revision already kept, so an unchanged flow is not rewritten on every tick. */ + private draftRevision = -1; + /** + * Which flow `draftId` belongs to. + * + * Without it, pressing Record on a finished recording kept the old draft id and wrote the new + * flow straight over the old one's file: the list showed one recording where the user had made + * two. `revision` cannot tell those apart — it moves for an edit and for a new flow alike — so + * the recorder mints a `flowId` and the draft is pinned to it. A flow that is not the one the + * draft was taken from is kept as a new draft, whatever the panel was holding a moment ago. + */ + private draftFlowId = ""; + private draftWrite?: Promise<void>; + /** Set while `dispose` makes its last keep, so the guards do not refuse the flush. */ + private flushing = false; + /** Set when a session goes active, for the host chrome to show the Testing tab. */ + private onAgentRecording?: () => void; + + constructor(app: HostBridge, deviceApps: DeviceAppsStore) { + this.app = app; + this.deviceApps = deviceApps; + // The device is half of what an agent recording needs and it belongs to the simulator panel, + // so the selection is reported whenever either half moves. + this.stopWatchingDevices = deviceApps.subscribe(() => { void this.reportContext(); }); + // Every change to what would be written bumps the recorder's revision, which is exactly the + // thing worth keeping, so one subscription covers recording, editing, dragging and the title. + this.stopWatchingRecorder = testRecorder.subscribe(() => { this.scheduleDraft(); }); + } + + /** + * The host chrome's way of being told to show the Testing tab. + * + * Which tool is on screen is the host's own state — the workspace component holds it — so the + * panel asks rather than reaches. Opening the tab is the point of an agent recording: the user + * asked their agent for a test and the steps have to be somewhere they can see them. + */ + onAgentRecordingStarted(listener: () => void) { + this.onAgentRecording = listener; + return () => { if (this.onAgentRecording === listener) this.onAgentRecording = undefined; }; + } + + subscribe = (listener: () => void) => { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }; + getSnapshot = () => this.snapshot; + + private update(fields: Partial<TestingSnapshot>) { + this.snapshot = { ...this.snapshot, ...fields }; + for (const listener of this.listeners) listener(); + this.syncTreeSource(); + void this.reportContext(); + } + + /** + * Keeps the recorder's tree source in step with the project and target. + * + * The source is what reads the e2e runner's own element tree, and reading it needs both: the + * project, whose `@e2e-dev/mobile` decides what every element's role is, and the target's bundle + * id, to attach to the app the user is already tapping. Without both there is nothing to read, + * and the recorder is told so rather than being left pointed at a stale project. + */ + private syncTreeSource() { + const projectPath = this.snapshot.projectPath; + const bundleId = this.target?.bundleId ?? ""; + const key = this.disposed || !projectPath || !bundleId ? "" : `${projectPath}\u0000${bundleId}`; + if (key === this.treeSourceKey) return; + this.treeSourceKey = key; + testRecorder.useTreeSource(key ? this.treeSource(projectPath, bundleId) : undefined); + } + + private treeSource(projectPath: string, bundleId: string): RecorderTreeSource { + return { + read: async (deviceId, platform) => { + const result = await this.call("mobile_testing_read_tree", { deviceId, platform, projectPath, bundleId }, 20000); + const content = object(result.structuredContent); + const tree = content?.tree; + // The screen the runner measured against. Android's is device pixels and the mirror's is + // the stream's own size, so the mirror cannot place these rectangles without it. + const screen = screenSize(content?.screen); + return { tree: Array.isArray(tree) ? tree as ScreenComponent[] : [], ...(screen ? { screen } : {}) }; + }, + // Fire and forget: giving the device back must never hold up stopping a recording, and a + // release that fails leaves the session for the next `read` to take over. + release: () => { void this.call("mobile_testing_release_device", {}, 10000).catch(() => undefined); }, + }; + } + + private async call(name: string, arguments_: Record<string, unknown>, timeout = 30000) { + const result = await this.app.callServerTool({ name, arguments: arguments_ }, { timeout }); + if (result.isError) throw textError(result, `${name} failed.`); + return result; + } + + show() { if (!this.disposed) { this.update({ open: true }); void this.restoreProject(); } } + hide() { this.update({ open: false }); } + setAvailable(available: boolean) { + this.update({ available }); + if (!available) return; + // Reopened on the last project whether or not the tab is showing: an agent recording needs a + // project the user has already chosen, and asking them to open the tab first only to have it + // reopen on the folder they picked last time is a step that answers nothing. The *choice* is + // still theirs — nothing is ever inferred from the server's working directory. + void this.restoreProject(); + // And the poll runs from here rather than from `show`, so a recording the agent starts while + // the user is in another tool is what opens the Testing tab for them. + this.pollAgent(); + void this.refreshDrafts(); + void this.refreshTests(); + } + + /** The target the recorder would use: the chosen one, else the first the config declares. */ + get target(): TestingTargetOption | undefined { + const { targets, targetName } = this.snapshot; + return targets.find(entry => entry.name === targetName) ?? targets[0]; + } + + /** + * Why the selected target cannot be recorded against the mirrored device, or "". A flow recorded + * on one platform and run on another is a test nobody can reproduce, so it is refused here + * rather than discovered when the run fails. + */ + get targetMismatch(): string { + const device = this.deviceApps.getSnapshot().device; + const target = this.target; + if (!device || !target?.platform) return ""; + const mirrored = recordablePlatform(device); + if (target.platform === mirrored) return ""; + return `Target "${target.name}" is ${target.platform}, but the mirror shows ${mirrored === "ios" ? "an iOS" : "an Android"} device. Choose ${mirrored === "ios" ? "an iOS" : "an Android"} target.`; + } + + /** Whether a launch is possible: a bundle id to launch and a booted device to launch it on. */ + get canLaunch(): boolean { + const device = this.deviceApps.getSnapshot().device; + const booted = device?.state === "Booted" || device?.state === "connected"; + return Boolean(this.snapshot.available && this.target?.bundleId && booted && !this.targetMismatch && !this.snapshot.launching); + } + + /** Reads the server's project list, so a pick survives the panel document. */ + async refreshProjects() { + this.projectRequest ??= (async () => { + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + const response = await this.call("mobile_testing_projects", {}); + const listed = object(response.structuredContent)?.projects; + const projects = (Array.isArray(listed) ? listed : []).map(entry => object(entry)) + .filter((entry): entry is Record<string, unknown> => entry !== null) + .filter(entry => text(entry.path)) + .map(entry => ({ path: text(entry.path), name: text(entry.name) || projectName(text(entry.path)) })); + if (!this.disposed) this.update({ projects }); + } catch { + // The panel still works from the project it was last opened with, or the empty state. + } finally { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.projects", performance.now() - startedAt); + this.projectRequest = undefined; + } + })(); + return this.projectRequest; + } + + /** Reopens the panel on the project it was last used with. Retryable: the host may not be up yet. */ + private async restoreProject() { + if (!this.snapshot.available || this.snapshot.projectPath || this.restoreRequest) return this.restoreRequest; + const request = (async () => { + await this.refreshProjects(); + const latest = this.snapshot.projects[0]; + if (latest && !this.disposed && !this.snapshot.projectPath) await this.selectProject(latest.path); + })(); + this.restoreRequest = request; + try { await request; } finally { if (this.restoreRequest === request) this.restoreRequest = undefined; } + } + + private remembered(project: TestingProjectOption) { + this.update({ projects: [project, ...this.snapshot.projects.filter(entry => entry.path !== project.path)] }); + } + + /** Selects a project the selector already lists, and makes it the one the panel reopens with. */ + async chooseProject(projectPath: string) { + const path = projectPath.trim(); + if (!path) return; + if (!this.snapshot.available) { await this.selectProject(path); return; } + try { + const response = await this.call("mobile_testing_choose_project", { projectPath: path }); + const project = object(object(response.structuredContent)?.project); + const chosen = text(project?.path); + if (!chosen) return; + this.remembered({ path: chosen, name: text(project?.name) || projectName(chosen) }); + await this.selectProject(chosen); + } catch (error) { + // The server refuses folders with no e2e config, so the current project stays put. + this.update({ error: error instanceof Error ? error.message : String(error) }); + } + } + + /** Opens the system folder picker. Cancelling keeps the current project. */ + async pickProject() { + if (this.snapshot.pickingProject) return; + this.update({ pickingProject: true, error: null }); + try { + // The wait is a person choosing a folder, so the server's dialog metric measures it. + const response = await this.call("mobile_testing_choose_project", {}, 300_000); + const project = object(object(response.structuredContent)?.project); + const path = text(project?.path); + if (!path) return; + this.remembered({ path, name: text(project?.name) || projectName(path) }); + await this.selectProject(path); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + this.update({ pickingProject: false }); + } + } + + /** Loads one project's e2e config into the panel. */ + async selectProject(projectPath: string) { + const path = projectPath.trim(); + const revision = ++this.loadRevision; + this.clearTestPreview(); + // The flow on screen belongs to the project it was recorded against, so the panel lets go of + // its draft rather than rewriting that draft under a different project's path. + clearTimeout(this.draftTimer); + this.draftRevision = testRecorder.getSnapshot().revision; + this.draftFlowId = ""; + this.update({ projectPath: path, targets: [], targetName: "", configPath: "", appLaunched: false, + overwritePath: "", savedRevision: -1, savedPath: "", draftId: "", drafts: [], tests: [], error: null }); + void this.refreshDrafts(); + void this.refreshTests(); + if (!path || !this.snapshot.available) return; + this.update({ loadingProject: true }); + try { + const response = await this.call("mobile_testing_project_config", { projectPath: path }); + if (revision !== this.loadRevision) return; + const data = object(response.structuredContent) ?? {}; + const listed = Array.isArray(data.targets) ? data.targets : []; + const targets = listed.map(entry => object(entry)).filter((entry): entry is Record<string, unknown> => entry !== null) + .filter(entry => text(entry.name)) + .map(entry => ({ + name: text(entry.name), + ...(text(entry.platform) ? { platform: text(entry.platform) } : {}), + ...(text(entry.bundleId) ? { bundleId: text(entry.bundleId) } : {}), + })); + // The target on the mirrored device's platform, so a project with both records against the + // right one without being asked; else the first, which `targetMismatch` then reports on. + const device = this.deviceApps.getSnapshot().device; + const preferred = (device ? targets.find(entry => entry.platform === recordablePlatform(device)) : undefined) ?? targets[0]; + this.update({ targets, targetName: preferred?.name ?? "", configPath: text(data.configPath) }); + } catch (error) { + if (revision === this.loadRevision) this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + if (revision === this.loadRevision) this.update({ loadingProject: false }); + } + } + + /** Records against another target the config declares. */ + selectTarget(name: string) { + if (!this.snapshot.targets.some(entry => entry.name === name)) return; + this.update({ targetName: name, appLaunched: false }); + } + + /** + * Launches the selected target's app on the mirrored device, so recording starts from the app + * the generated test will open. It does not build: an app that is not installed fails here, and + * the message says so. + */ + async launchApp() { + const device = this.deviceApps.getSnapshot().device; + const bundleId = this.target?.bundleId; + if (!device || !bundleId || this.snapshot.launching) return; + this.update({ launching: true, error: null }); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + await this.call("mobile_launch_app", { + platform: recordablePlatform(device), + deviceId: device.udid, + bundleId, + }, 120_000); + this.update({ appLaunched: true }); + countUiEvent("ui.testing.launch"); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + captureUiError(error, "testing.launch", telemetryContext); + } finally { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.launch", performance.now() - startedAt); + this.update({ launching: false }); + } + } + + /** + * The unsaved recordings of the chosen project, newest first. + * + * Listed for the project rather than for everything: a draft is reopened into a panel pointed + * at one project, and a flow recorded against another app's selectors would come back naming + * elements that project has never had. + */ + async refreshDrafts() { + if (!this.snapshot.available) return; + try { + const response = await this.call("mobile_testing_drafts", + this.snapshot.projectPath ? { projectPath: this.snapshot.projectPath } : {}, 10_000); + const listed = object(response.structuredContent)?.drafts; + const drafts = (Array.isArray(listed) ? listed : []) as TestDraftSummary[]; + if (!this.disposed) this.update({ drafts }); + } catch { + // The list is a convenience; the flow on screen is unaffected by a read that missed. + } + } + + /** + * The tests the chosen project already holds. + * + * Read from the folder rather than remembered: a test written by hand, by an agent or in another + * checkout belongs in the list beside the recorded ones, and a recording that was saved has to + * appear here the moment its draft goes. + */ + async refreshTests() { + if (!this.snapshot.available || !this.snapshot.projectPath) { + this.clearTestPreview(); + if (this.snapshot.tests.length > 0) this.update({ tests: [] }); + return; + } + const projectPath = this.snapshot.projectPath; + try { + const response = await this.call("mobile_testing_tests", { projectPath }, 15_000); + const listed = object(response.structuredContent)?.tests; + const tests = (Array.isArray(listed) ? listed : []) as StoredTestSummary[]; + // A slow read must not land on a project the user has since left. + if (!this.disposed && this.snapshot.projectPath === projectPath) { + if (this.snapshot.selectedTestPath && !tests.some(test => test.relativePath === this.snapshot.selectedTestPath)) this.clearTestPreview(); + this.update({ tests }); + } + } catch { + // The list is a readout; a read that missed leaves the recording on screen untouched. + } + } + + private clearTestPreview() { + ++this.testReadRevision; + if (this.snapshot.selectedTestPath) testRecorder.clearRun(); + if (this.snapshot.selectedTestPath || this.snapshot.loadingTest) this.update({ selectedTestPath: "", selectedTestSource: "", selectedTestOriginalHash: "", selectedTestStaged: false, loadingTest: false }); + } + + /** Shows the file exactly as stored. The recorder's current flow remains intact. */ + async openTest(relativePath: string) { + const projectPath = this.snapshot.projectPath; + if (!relativePath || !projectPath || this.snapshot.agent?.active) return; + const revision = ++this.testReadRevision; + clearTimeout(this.draftTimer); + await this.keepDraft(); + await this.stageWrite?.catch(() => undefined); + if (this.disposed || this.snapshot.projectPath !== projectPath || revision !== this.testReadRevision) return; + testRecorder.stop(); + testRecorder.clearRun(); + this.update({ selectedTestPath: relativePath, selectedTestSource: "", loadingTest: true, error: null }); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + const response = await this.call("mobile_testing_read_file", { projectPath, relativePath }, 15_000); + if (revision === this.testReadRevision && this.snapshot.projectPath === projectPath) { + const data = object(response.structuredContent); + this.update({ selectedTestSource: text(data?.source), selectedTestOriginalHash: text(data?.originalHash), selectedTestStaged: data?.staged === true }); + } + countUiEvent("ui.testing.test_preview"); + } catch (error) { + if (revision === this.testReadRevision) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + captureUiError(error, "testing.test_preview", telemetryContext); + } + } finally { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.test_preview", performance.now() - startedAt); + if (revision === this.testReadRevision) this.update({ loadingTest: false }); + } + } + + /** Edits the temporary working copy; the project file is only replaced by Save. */ + editStoredTest(source: string) { + const { projectPath, selectedTestPath: relativePath, selectedTestOriginalHash: originalHash } = this.snapshot; + if (!projectPath || !relativePath || !originalHash || this.snapshot.running || this.snapshot.saving || source === this.snapshot.selectedTestSource) return; + this.update({ selectedTestSource: source, selectedTestStaged: true, error: null }); + testRecorder.clearRun(); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + const write = (this.stageWrite ?? Promise.resolve()).catch(() => undefined).then(async () => { + await this.call("mobile_testing_stage_file", { projectPath, relativePath, originalHash, source }, 15_000); + countUiEvent("ui.testing.test_edit"); + }); + this.stageWrite = write; + void write.catch(error => { + if (this.snapshot.projectPath === projectPath && this.snapshot.selectedTestPath === relativePath) this.update({ error: error instanceof Error ? error.message : String(error) }); + captureUiError(error, "testing.test_stage", telemetryContext); + }).finally(() => { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.test_stage", performance.now() - startedAt); + }); + } + + async saveStoredTest() { + const { projectPath, selectedTestPath: relativePath } = this.snapshot; + if (!projectPath || !relativePath || !this.snapshot.selectedTestStaged || this.snapshot.saving) return; + this.update({ saving: true, error: null }); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + await this.stageWrite; + const response = await this.call("mobile_testing_save_staged_file", { projectPath, relativePath }, 60_000); + if (this.snapshot.projectPath === projectPath && this.snapshot.selectedTestPath === relativePath) + this.update({ selectedTestStaged: false, selectedTestOriginalHash: text(object(response.structuredContent)?.originalHash) }); + countUiEvent("ui.testing.test_save"); + await this.refreshTests(); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + captureUiError(error, "testing.test_save", telemetryContext); + } finally { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.test_save", performance.now() - startedAt); + this.update({ saving: false }); + } + } + + async undoStoredTest() { + const { projectPath, selectedTestPath: relativePath } = this.snapshot; + if (!projectPath || !relativePath || !this.snapshot.selectedTestStaged || this.snapshot.saving) return; + this.update({ saving: true, error: null }); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + await this.stageWrite?.catch(() => undefined); + const response = await this.call("mobile_testing_discard_staged_file", { projectPath, relativePath }, 15_000); + if (this.snapshot.projectPath === projectPath && this.snapshot.selectedTestPath === relativePath) { + const data = object(response.structuredContent); + this.update({ selectedTestSource: text(data?.source), selectedTestOriginalHash: text(data?.originalHash), selectedTestStaged: false }); + testRecorder.clearRun(); + } + countUiEvent("ui.testing.test_undo"); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + captureUiError(error, "testing.test_undo", telemetryContext); + } finally { + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.test_undo", performance.now() - startedAt); + this.update({ saving: false }); + } + } + + /** + * Deletes one test out of the project. The panel asks first — see the row's confirmation — and + * the host asks again, because this removes a file from the user's repository. + */ + async deleteTest(relativePath: string) { + const projectPath = this.snapshot.projectPath; + if (!relativePath || !projectPath || !this.snapshot.available || this.snapshot.deletingTest) return; + this.update({ deletingTest: relativePath, error: null }); + try { + await this.call("mobile_testing_delete_file", { projectPath, relativePath }, 15_000); + countUiEvent("ui.testing.test_delete"); + // The file the panel last wrote is the one that just went, so the "Saved to …" line under + // the editor would be pointing at nothing. + if (this.snapshot.savedPath.endsWith(relativePath)) this.update({ savedPath: "", savedRevision: -1 }); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + this.update({ deletingTest: "" }); + } + await this.refreshTests(); + } + + /** Asks for a keep after the edits settle. Called on every recorder change. */ + private scheduleDraft() { + if (this.disposed) return; + // A new flow is not the kept one, and saying so here rather than at the next write keeps the + // picker honest: the row it ticks and the name on its button are about the flow on screen. + if (this.snapshot.draftId && testRecorder.getSnapshot().flowId !== this.draftFlowId) { + this.update({ draftId: "" }); + } + clearTimeout(this.draftTimer); + this.draftTimer = setTimeout(() => { void this.keepDraft(); }, DRAFT_AUTOSAVE_MS); + } + + /** + * Keeps the flow on screen as an unsaved recording. + * + * Not while an agent is recording: the server holds that flow and writes its own draft when the + * session ends, so keeping it here too would leave two copies of one recording in the list. Not + * once it has been written into the project either — the draft is deleted at that point, and + * the panel has nothing left to protect. + */ + private async keepDraft(): Promise<void> { + const { available, projectPath, agent } = this.snapshot; + const recorder = testRecorder.getSnapshot(); + if ((this.disposed && !this.flushing) || !available || !projectPath || agent?.active) return; + if ((recorder.steps.length === 0 && !recorder.sourceOverride) || recorder.revision === this.draftRevision) return; + if (recorder.revision === this.snapshot.savedRevision) return; + const revision = recorder.revision; + // The draft only belongs to the flow it was taken from. Record, Clear and an agent session + // all begin a different recording, and writing this one over that one's file is how two + // recordings became one. + const sameFlow = recorder.flowId === this.draftFlowId; + const replacing = sameFlow ? this.snapshot.draftId : ""; + const flowId = recorder.flowId; + // Serialized: two writes in flight would race over which one the list ends on, and the later + // call carries the newer steps anyway. + const write = (this.draftWrite ?? Promise.resolve()).catch(() => undefined).then(async () => { + if (this.disposed && !this.flushing) return; + // Not measured while the panel is being torn down: the dispose-time keep is bookkeeping on + // the way out, not a wait the user sat through, and a measurement recorded from a document + // that is going away has no surface left to attribute it to. + const measured = !this.flushing; + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + try { + const response = await this.call("mobile_testing_draft_save", { + ...(replacing ? { id: replacing } : {}), + projectPath, title: recorder.title, fileName: this.snapshot.fileName, + origin: recorder.agentDeviceId ? "agent" : "user", + steps: recorder.steps, + ...(recorder.sourceOverride === undefined ? {} : { source: recorder.sourceOverride }), + }, 15_000); + const id = text(object(response.structuredContent)?.id); + this.draftRevision = revision; + this.draftFlowId = flowId; + if (this.disposed) return; + if (id) this.update({ draftId: id }); + await this.refreshDrafts(); + } catch (error) { + // Silent: an autosave that failed must not put a banner over a recording that is working. + // The next change tries again, and the steps are still on screen either way. + if (measured) captureUiError(error, "testing.draft_save", telemetryContext); + } finally { + if (measured && getUiTelemetryAttributes() === telemetryContext) { + recordUiTiming("ui.testing.draft_save", performance.now() - startedAt); + } + } + }); + this.draftWrite = write; + await write; + } + + /** + * Puts one kept recording back in the panel. + * + * The flow on screen is kept first, so swapping between two drafts never loses the one being + * left. Nothing starts recording: the user opened a flow to look at it. + */ + async openDraft(id: string) { + if (!id || this.snapshot.loadingDraft || this.snapshot.agent?.active) return; + this.clearTestPreview(); + if (id === this.snapshot.draftId) return; + clearTimeout(this.draftTimer); + await this.keepDraft(); + if (this.disposed) return; + this.update({ loadingDraft: true, error: null }); + try { + const response = await this.call("mobile_testing_draft_read", { id }, 15_000); + const draft = object(object(response.structuredContent)?.draft); + const steps = Array.isArray(draft?.steps) ? draft.steps as RecordedStep[] : []; + if (!draft || (steps.length === 0 && !text(draft.source))) { + this.update({ error: "That recording is no longer there." }); + await this.refreshDrafts(); + return; + } + testRecorder.restore(steps, text(draft.title) || undefined, typeof draft.source === "string" ? draft.source : undefined); + // The restored flow is already kept, so the change `restore` just made is not written back, + // and the draft is pinned to the flow it came back as. + this.draftRevision = testRecorder.getSnapshot().revision; + this.draftFlowId = testRecorder.getSnapshot().flowId; + this.update({ draftId: text(draft.id), fileName: text(draft.fileName), savedRevision: -1, savedPath: "", overwritePath: "" }); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + this.update({ loadingDraft: false }); + } + } + + /** + * Throws one kept recording away. Deleting the one on screen clears the panel too: leaving the + * steps up after the user deleted them would only have the next edit write them back. + */ + async deleteDraft(id: string) { + if (!id || !this.snapshot.available) return; + const current = id === this.snapshot.draftId; + if (current && this.snapshot.agent?.active) return; + clearTimeout(this.draftTimer); + try { + await this.call("mobile_testing_draft_delete", { id }, 15_000); + countUiEvent("ui.testing.draft_delete"); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + return; + } + if (current) this.forgetDraft(true); + await this.refreshDrafts(); + } + + /** + * Lets go of the draft the panel is holding, optionally clearing the steps with it. Used when a + * flow is written into the project and when its draft is deleted. + */ + private forgetDraft(clearSteps: boolean) { + clearTimeout(this.draftTimer); + this.draftRevision = -1; + this.draftFlowId = ""; + if (clearSteps) { + testRecorder.stop(); + testRecorder.clearSteps(); + this.draftRevision = testRecorder.getSnapshot().revision; + } + this.update({ draftId: "", fileName: clearSteps ? "" : this.snapshot.fileName }); + } + + /** Starts a new flow, keeping whatever is on screen as its own draft first. */ + async newRecording() { + if (this.snapshot.agent?.active) return; + this.clearTestPreview(); + clearTimeout(this.draftTimer); + await this.keepDraft(); + if (this.disposed) return; + testRecorder.stop(); + testRecorder.clearSteps(); + testRecorder.setTitle(""); + this.draftRevision = testRecorder.getSnapshot().revision; + this.draftFlowId = ""; + this.update({ draftId: "", fileName: "", savedRevision: -1, savedPath: "", overwritePath: "" }); + } + + setFileName(fileName: string) { this.update({ fileName: fileName.trim(), overwritePath: "" }); } + + /** The path the Save button writes to, relative to the project. */ + get relativePath(): string { + const base = (this.snapshot.fileName || testFileSlug(testRecorder.getSnapshot().title)).replace(/\.e2e\.ts$/, ""); + return `tests/${base}.e2e.ts`; + } + + /** Whether the recording has changes that are not in the file yet. */ + get dirty(): boolean { + const recorder = testRecorder.getSnapshot(); + return recorder.steps.length > 0 && recorder.revision !== this.snapshot.savedRevision; + } + + /** + * Writes the recorded test into the project. The file is only ever written here — nothing is + * saved as you tap — so one save is one host permission prompt. + * + * An existing file is not replaced on the first attempt: the server reports it back and the + * panel asks. `confirmOverwrite` is the user's answer to that question. + */ + async saveTest(confirmOverwrite = false) { + const projectPath = this.snapshot.projectPath; + const recorder = testRecorder.getSnapshot(); + if (!projectPath || !this.snapshot.available || this.snapshot.saving) return; + if (recorder.steps.length === 0 && !recorder.sourceOverride) { this.update({ error: "Record a step before saving." }); return; } + const relativePath = this.relativePath; + this.update({ saving: true, error: null }); + try { + const response = await this.call("mobile_testing_write_file", { + projectPath, relativePath, contents: testRecorder.source, overwrite: confirmOverwrite, + }, 60_000); + const data = object(response.structuredContent) ?? {}; + if (data.written === true) { + const resume = this.snapshot.pendingRun; + const keptAs = this.snapshot.draftId; + this.update({ savedRevision: recorder.revision, savedPath: text(data.path), overwritePath: "", pendingRun: false }); + // The flow is in the project now, which is what the draft was standing in for. The steps + // stay on screen; only the copy in the plugin's own storage goes. + if (keptAs) { + this.forgetDraft(false); + void this.call("mobile_testing_draft_delete", { id: keptAs }, 10_000) + .then(() => this.refreshDrafts()) + .catch(() => undefined); + } + // And it is one of the project's tests now, so it appears in the stored half of the list + // rather than leaving the list with its draft. + void this.refreshTests(); + // The flow is written, so the recording is over: leaving it armed turns the next stray tap + // on the mirror into a step the saved file does not have. + testRecorder.stop(); + countUiEvent("ui.testing.save"); + // Replacing the file was the answer Run was waiting for, so one click finishes what it began. + if (resume) { this.update({ saving: false }); void this.runTest(); } + return; + } + // The file is already there and the user has not said to replace it. + this.update({ overwritePath: text(data.path) || relativePath }); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + } finally { + this.update({ saving: false }); + } + } + + cancelOverwrite() { this.update({ overwritePath: "", pendingRun: false }); } + + /** + * One poll of the run's live progress. Failures are silent: the run is still going, the report + * will say what happened, and a progress read that missed is not worth an error banner. + */ + private async pollRunProgress(callLines?: readonly number[]) { + try { + const response = await this.call("mobile_testing_run_progress", {}, 5_000); + const data = object(response.structuredContent) ?? {}; + const entries = (Array.isArray(data.progress) ? data.progress : []).map(item => object(item)) + .filter((item): item is Record<string, unknown> => item !== null) + .filter(item => typeof item.stepIndex === "number" && (item.phase === "start" || item.phase === "end")) + .map(item => ({ + stepIndex: item.stepIndex as number, + phase: item.phase as "start" | "end", + ...(text(item.status) ? { status: text(item.status) } : {}), + ...(text(item.error) ? { error: text(item.error) } : {}), + })); + testRecorder.applyRunProgress(entries, callLines); + } catch { + // See above. + } + } + + /** + * Tells the server what the panel has selected, so an agent recording has something to record + * against. + * + * This is the only thing that knows the selection: the project is the user's explicit pick, the + * device is whichever one the mirror shows, and the bundle id comes from the project's own e2e + * config. `mobile_testing_agent_begin` reads it and refuses when any of the three is missing, + * which is what keeps an OS folder dialog from opening inside an agent's turn. + */ + private async reportContext() { + if (this.disposed || !this.snapshot.available) return; + const projectPath = this.snapshot.projectPath; + const bundleId = this.target?.bundleId ?? ""; + const device = this.deviceApps.getSnapshot().device; + // Only a booted iOS simulator or Android emulator can be recorded against; anything else is + // reported as no device. + const recordable = device !== undefined && isRecordableDevice(device); + const deviceId = recordable ? device.udid : ""; + const platform = recordable ? recordablePlatform(device) : undefined; + const key = `${projectPath}\u0000${platform ?? ""}\u0000${deviceId}\u0000${bundleId}`; + if (key === this.reportedContext) return; + this.reportedContext = key; + try { + await this.call("mobile_testing_set_context", { projectPath, deviceId, ...(platform ? { platform } : {}), bundleId }, 10_000); + } catch { + // The next change reports again; a selection the server missed only means an agent is told + // to ask the user, which is the refusal it would have given anyway. + this.reportedContext = ""; + } + } + + /** + * One poll of the agent recording the server holds, which then schedules the next. + * + * Fast while a session is active so a step appears in the panel as it is recorded, slow + * otherwise so a session started from another tool is still noticed. Failures are silent and do + * not stop the loop: a missed tick is one the next one covers. + */ + private pollAgent() { + if (this.disposed || this.agentPolling) return; + this.agentPolling = true; + void this.readAgentState().finally(() => { + this.agentPolling = false; + if (this.disposed || !this.snapshot.available) return; + clearTimeout(this.agentTimer); + this.agentTimer = setTimeout(() => this.pollAgent(), + this.snapshot.agent?.active ? AGENT_POLL_ACTIVE_MS : AGENT_POLL_IDLE_MS); + }); + } + + private async readAgentState() { + let data: Record<string, unknown>; + try { + const response = await this.call("mobile_testing_agent_state", {}, 5_000); + data = object(response.structuredContent) ?? {}; + } catch { + return; + } + if (this.disposed) return; + const revision = typeof data.revision === "number" ? data.revision : 0; + const status = data.status === "recording" || data.status === "finished" || data.status === "aborted" ? data.status : "idle"; + const steps = (Array.isArray(data.steps) ? data.steps : []) as RecordedStep[]; + const agent: TestingAgentView = { + active: data.active === true, status, + title: text(data.title), goal: text(data.goal), steps: steps.length, note: text(data.note), + draftId: text(data.draftId), + }; + const first = this.agentRevision < 0; + const unchanged = !first && revision === this.agentRevision; + this.agentRevision = revision; + if (status === "idle") { this.update({ agent: null }); return; } + const wasActive = this.snapshot.agent?.active === true; + this.update({ agent }); + if (agent.active) { + this.clearTestPreview(); + // A session already in flight when the panel opened is adopted too: the agent may have + // started before the user's host drew this document. + if (!unchanged || !wasActive) { + const device = this.deviceApps.getSnapshot().device; + if (device) testRecorder.adoptAgentRecording(device.udid, recordablePlatform(device), steps, agent.title); + } + if (!wasActive) this.onAgentRecording?.(); + return; + } + // The agent stopped. The steps are left dirty exactly as they are, and from here the flow is + // an ordinary unsaved recording: the banner clears and the mirror is the user's again. + if (wasActive) { + if (steps.length !== testRecorder.getSnapshot().steps.length) { + const device = this.deviceApps.getSnapshot().device; + if (device) testRecorder.adoptAgentRecording(device.udid, recordablePlatform(device), steps, agent.title); + } + testRecorder.endAgentRecording(status === "aborted" ? "aborted" : "finished"); + // The server kept the finished flow, so the panel adopts that draft rather than autosaving + // a second copy of the same steps a moment later. + this.draftRevision = testRecorder.getSnapshot().revision; + this.draftFlowId = agent.draftId ? testRecorder.getSnapshot().flowId : ""; + this.update({ savedRevision: -1, savedPath: "", ...(agent.draftId ? { draftId: agent.draftId } : {}) }); + void this.refreshDrafts(); + } + } + + /** The user's Stop button on the agent banner. The server drops the session; the steps stay. */ + async stopAgent() { + if (!this.snapshot.agent?.active) return; + try { + await this.call("mobile_testing_agent_abort", { reason: "The user stopped the recording from the Testing panel." }, 10_000); + } catch (error) { + this.update({ error: error instanceof Error ? error.message : String(error) }); + return; + } + // Do not wait for the next tick to hand the mirror back: the user pressed Stop to get it back. + testRecorder.endAgentRecording("aborted"); + this.update({ agent: { ...this.snapshot.agent, active: false, status: "aborted" } }); + this.pollAgent(); + } + + /** Whether Run has something to run: a project, steps, and no run or save already in flight. */ + get canRun(): boolean { + // Never while an agent is recording: a run resets the app and takes the one agent-device + // session the device allows, which is the session the recording is being read through. + return Boolean(this.snapshot.available && this.snapshot.projectPath && !this.snapshot.selectedTestPath && !this.snapshot.agent?.active + && (testRecorder.getSnapshot().steps.length > 0 || Boolean(testRecorder.getSnapshot().sourceOverride)) && !this.snapshot.running && !this.snapshot.saving); + } + + get canRunStored(): boolean { + return Boolean(this.snapshot.available && this.snapshot.projectPath && this.snapshot.selectedTestPath + && !this.snapshot.loadingTest && !this.snapshot.agent?.active && !this.snapshot.running && !this.snapshot.saving); + } + + /** Runs a temporary sibling copy so Play and Run never save the working copy implicitly. */ + async runStoredTest(untilLine?: number) { + if (!this.canRunStored) return; + const { projectPath, selectedTestPath: relativePath, selectedTestSource: source } = this.snapshot; + const untilSource = untilLine === undefined ? source : storedTestUntilSource(source, untilLine); + if (untilSource === undefined) return; + this.update({ running: true, error: null }); + testRecorder.clearRun(); + testRecorder.beginRun(untilLine); + if (untilLine !== undefined) countUiEvent("ui.testing.run_until"); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + // A saved project test has no recorder steps to supply `emitted.calls`, so map the runner's + // step indexes against the awaited calls in the source currently shown in the preview. + const callLines = storedTestCallLines(source); + const poll = setInterval(() => { void this.pollRunProgress(callLines); }, 400); + try { + await this.stageWrite; + await this.call("mobile_testing_release_device", {}, 30_000).catch(() => undefined); + const response = await this.call("mobile_testing_run_test", { + projectPath, relativePath, untilSource, + ...(this.snapshot.configPath ? { configPath: this.snapshot.configPath } : {}), + ...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}), + }, 15 * 60_000); + const data = object(response.structuredContent) ?? {}; + const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry)) + .filter((entry): entry is Record<string, unknown> => entry !== null) + .map(entry => ({ line: typeof entry.line === "number" ? entry.line : 0, status: text(entry.status), ...(text(entry.error) ? { error: text(entry.error) } : {}) })) + .filter(step => step.line > 0); + testRecorder.finishRun({ status: text(data.status) || "failed", steps, + ...(text(data.output) ? { output: text(data.output) } : {}), + ...(typeof data.durationMs === "number" ? { durationMs: data.durationMs } : {}) }); + countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + testRecorder.failRun(message); + this.update({ error: message }); + captureUiError(error, "testing.run", telemetryContext); + } finally { + clearInterval(poll); + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.run", performance.now() - startedAt); + this.update({ running: false }); + testRecorder.invalidate(); + } + } + + /** + * Runs the recorded test against the device, and marks the preview with what the runner reports. + * + * **Run saves.** The runner runs a file, so there has to be one; running a copy in a temporary + * path would run a test whose failures point at lines of a file the user cannot open. It saves + * through the same path the Save button uses, which means an existing file is still never + * replaced without being asked about — the overwrite question stops the run and resumes it from + * the answer. + * + * **Run releases the device first.** The recorder holds the one agent-device session the device + * allows, so a run started while it is held fails with `Device is already in use by session + * "mobile-dev-recorder"` — a refusal the user cannot connect to anything they did. Recording + * stops, the session is given back and *waited for*, and only then is the runner spawned. The + * server tool releases again on its own account, because a release the caller forgot is the + * same failure. + * + * **Play until here does not save.** `untilStepId` runs the flow only as far as that step, from a + * truncated copy the server writes beside the test and deletes when the run ends — so the saved + * file is never cut down to the part that was tried, and a half-run never raises the overwrite + * question about a file the user has not asked to replace. Every line the copy keeps is at the + * line number it has in the full file, so the marks land on the preview unchanged; see + * `emittedThrough` and `untilSource`. + */ + async runTest(options: { untilStepId?: string } = {}) { + if (!this.canRun) return; + const until = options.untilStepId ? testRecorder.emittedThrough(options.untilStepId) : undefined; + // The step went while the pointer was on its play button; there is nothing to run up to. + if (options.untilStepId && !until) return; + if (!until && (this.dirty || !this.snapshot.savedPath)) { + await this.saveTest(); + // The file is already there and the user has not answered yet; `saveTest(true)` resumes. + if (this.snapshot.overwritePath) { this.update({ pendingRun: true }); return; } + if (this.dirty) return; + } + const relativePath = this.relativePath; + this.update({ running: true, error: null, pendingRun: false }); + testRecorder.clearRun(); + // A recording left armed would read the tree mid-run and take the session straight back. + testRecorder.stop(); + try { + await this.call("mobile_testing_release_device", {}, 30_000); + } catch { + // A release that fails is still worth running through: the tool releases again before it + // spawns, and the runner's own "device is already in use" is a clearer report than ours. + } + testRecorder.beginRun(until?.untilLine); + if (until) countUiEvent("ui.testing.run_until"); + const startedAt = performance.now(), telemetryContext = getUiTelemetryAttributes(); + // The run's own tool call is one request and one response, so progress comes from polling a + // second tool while it is in flight. See `mobile_testing_run_progress`. + const poll = setInterval(() => { void this.pollRunProgress(); }, 400); + try { + const response = await this.call("mobile_testing_run_test", { + projectPath: this.snapshot.projectPath, relativePath, + ...(this.snapshot.configPath ? { configPath: this.snapshot.configPath } : {}), + ...(this.snapshot.targetName ? { target: this.snapshot.targetName } : {}), + ...(until ? { untilSource: until.source } : {}), + }, 15 * 60_000); + const data = object(response.structuredContent) ?? {}; + const steps = (Array.isArray(data.steps) ? data.steps : []).map(entry => object(entry)) + .filter((entry): entry is Record<string, unknown> => entry !== null) + .map(entry => ({ + line: typeof entry.line === "number" ? entry.line : 0, + status: text(entry.status), + ...(text(entry.error) ? { error: text(entry.error) } : {}), + })) + .filter(step => step.line > 0); + // The runner's own number for the run, which is not the timing below: that one brackets the + // whole call, spawning the CLI and resolving the project included. + const durationMs = typeof data.durationMs === "number" && Number.isFinite(data.durationMs) ? data.durationMs : undefined; + testRecorder.finishRun({ + status: text(data.status) || "failed", steps, + ...(text(data.output) ? { output: text(data.output) } : {}), + ...(durationMs === undefined ? {} : { durationMs }), + }); + countUiEvent(text(data.status) === "passed" ? "ui.testing.run_passed" : "ui.testing.run_failed"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + testRecorder.failRun(message); + this.update({ error: message }); + captureUiError(error, "testing.run", telemetryContext); + } finally { + clearInterval(poll); + if (getUiTelemetryAttributes() === telemetryContext) recordUiTiming("ui.testing.run", performance.now() - startedAt); + this.update({ running: false }); + // The run drove the app, so whatever the recorder cached describes a screen that has gone. + testRecorder.invalidate(); + } + } + + dismissError() { this.update({ error: null }); } + + dispose() { + // One last keep before the document goes: the host rebuilds this panel whenever the user + // looks at another tool, which is the very moment a recording used to disappear, and the + // debounce may still be holding the newest step. + this.flushing = true; + void this.keepDraft().finally(() => { this.flushing = false; }); + this.disposed = true; + clearTimeout(this.agentTimer); + clearTimeout(this.draftTimer); + this.stopWatchingDevices(); + this.stopWatchingRecorder(); + this.update({ open: false }); + this.listeners.clear(); + // The panel is going; the device it was reading must not go with it. See `RecorderTreeSource`. + this.syncTreeSource(); + } +} diff --git a/packages/ui/src/workspace-devices.ts b/packages/ui/src/workspace-devices.ts index 0aa67a80..743c2368 100644 --- a/packages/ui/src/workspace-devices.ts +++ b/packages/ui/src/workspace-devices.ts @@ -14,6 +14,17 @@ export class WorkspaceDevices { private loading?: Promise<void>; private disposed = false; private manualLayout = false; + /** + * The layout this chose for the user, so an unresolved one can still be finished. + * + * The automatic layout used to be decided exactly once, from the **first** device list. A list + * that came back before the backend had found anything — which is the ordinary case, since the + * opening tool starts the backend — resolved to "none", both panels stayed hidden, and nothing + * ever looked again: the user had a booted simulator, an empty workspace, a platform selector + * reading "Platforms", and a panel telling them to boot a simulator. Discovery finds the device + * a moment later and the panels select it, so what was missing was somebody asking again. + */ + private reported?: DeviceLayout; constructor(panels: readonly [SimulatorPanel, SimulatorPanel], onLayout: (layout: DeviceLayout) => void) { this.panels = panels; @@ -26,7 +37,7 @@ export class WorkspaceDevices { if (this.manualLayout === false) { for (const panel of this.panels) panel.setOpeningTarget(target); const platform = panelTargetPlatform(target); - if (platform !== undefined) this.onLayout(platform); + if (platform !== undefined) this.report(platform); } return this.load(); } @@ -42,6 +53,27 @@ export class WorkspaceDevices { for (const panel of this.panels) panel.setOpeningTarget(undefined); } + /** + * Finishes an automatic layout that resolved to "none", once a panel has a device to show. + * + * Called whenever a panel's selection changes, which is what device discovery does when it finds + * the simulator the first list missed. It only ever turns "none" into a platform: a layout that + * resolved to something, and any layout the user chose themselves, are left exactly as they are. + */ + settleLayout() { + // An unresolved layout is one that was never reported, or was reported as "none". + if (this.disposed || this.manualLayout) return; + if (this.reported !== undefined && this.reported !== "none") return; + const [ios, android] = this.panels; + const layout = activeDeviceLayout(ios.selected, android.selected); + if (layout !== "none") this.report(layout); + } + + private report(layout: DeviceLayout) { + this.reported = layout; + this.onLayout(layout); + } + dispose() { this.disposed = true; } private load(): Promise<void> { @@ -62,6 +94,6 @@ export class WorkspaceDevices { if (layout === "none" && (target.kind !== undefined || target.deviceId !== undefined)) { layout = ios.selected ? "ios" : android.selected ? "android" : "both"; } - this.onLayout(layout); + this.report(layout); } } diff --git a/packages/ui/src/workspace-tools.ts b/packages/ui/src/workspace-tools.ts index 6407cdb0..344a5cf0 100644 --- a/packages/ui/src/workspace-tools.ts +++ b/packages/ui/src/workspace-tools.ts @@ -6,14 +6,15 @@ import { PanelContext } from "./model-context.ts"; import { PerformancePanel } from "./performance-panel.ts"; import { StoragePanel } from "./storage-panel.ts"; import { markUiSurfaceReady, setUiSurface } from "./telemetry.ts"; +import { TestingPanel } from "./testing-panel.ts"; import { TraceSiftPanel } from "./tracesift-panel.ts"; import { AppIntelligencePanel } from "./app-intelligence-panel.ts"; -export const toolNames = ["logs", "performance", "storage", "app-intel", "tracesift", "app-flow"] as const; +export const toolNames = ["logs", "performance", "storage", "app-intel", "tracesift", "app-flow", "testing"] as const; export type ToolName = typeof toolNames[number]; export const isToolName = (value: unknown): value is ToolName => toolNames.includes(value as ToolName); -type Panels = { context: PanelContext; deviceApps: DeviceAppsStore; logs: LogsPanel; performance: PerformancePanel; storage: StoragePanel; "app-intel": AppIntelligencePanel; tracesift: TraceSiftPanel; "app-flow": AppFlowPanel }; +type Panels = { context: PanelContext; deviceApps: DeviceAppsStore; logs: LogsPanel; performance: PerformancePanel; storage: StoragePanel; "app-intel": AppIntelligencePanel; tracesift: TraceSiftPanel; "app-flow": AppFlowPanel; testing: TestingPanel }; /** The tool panels every host shows beside its devices; hosts only choose the selected tool and draw chrome. */ export class WorkspaceTools { @@ -25,6 +26,7 @@ export class WorkspaceTools { readonly "app-intel": AppIntelligencePanel; readonly tracesift: TraceSiftPanel; readonly "app-flow": AppFlowPanel; + readonly testing: TestingPanel; /** Builds the panels against one host; `attachments` reads a host's composer attachments, if it has them. */ static create(host: HostBridge, attachments?: HostAttachmentsSource) { @@ -38,11 +40,12 @@ export class WorkspaceTools { "app-intel": new AppIntelligencePanel(host), tracesift: new TraceSiftPanel(host, attachments), "app-flow": new AppFlowPanel(host), + testing: new TestingPanel(host, deviceApps), }); } constructor(panels: Panels) { - ({ context: this.context, deviceApps: this.deviceApps, logs: this.logs, performance: this.performance, storage: this.storage, "app-intel": this["app-intel"], tracesift: this.tracesift, "app-flow": this["app-flow"] } = panels); + ({ context: this.context, deviceApps: this.deviceApps, logs: this.logs, performance: this.performance, storage: this.storage, "app-intel": this["app-intel"], tracesift: this.tracesift, "app-flow": this["app-flow"], testing: this.testing } = panels); } panel(name: ToolName) { return this[name]; } @@ -85,6 +88,7 @@ export class WorkspaceTools { this.deviceApps.dispose(); this.storage.dispose(); this.tracesift.dispose(); + this.testing.dispose(); this["app-intel"].dispose(); await Promise.allSettled([this.logs.dispose(), this.performance.dispose(), this["app-flow"].dispose()]); } diff --git a/packages/ui/test/assertion-menu-ui.test.ts b/packages/ui/test/assertion-menu-ui.test.ts new file mode 100644 index 00000000..c8e4d3a8 --- /dev/null +++ b/packages/ui/test/assertion-menu-ui.test.ts @@ -0,0 +1,72 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { build } from "esbuild"; +import { JSDOM } from "jsdom"; + +test("assertion row inserts directly while its arrow offers an explicit timeout", async t => { + const built = await build({ + stdin: { contents: ` + import { createRoot } from "react-dom/client"; + import { flushSync } from "react-dom"; + import { AssertionBranch } from "./packages/ui/src/components/step-commands.tsx"; + export function mount() { + const chosen = []; + const root = createRoot(document.getElementById("root")); + flushSync(() => root.render(<> + <AssertionBranch offer={{ kind: "assertVisible", source: "expect(locator).toBeVisible()" }} + onChoose={(...args) => chosen.push(args)} /> + <AssertionBranch offer={{ kind: "assertText", source: "expect(locator).toHaveText()", needsValue: true, suggestedValue: "Hello" }} + onChoose={(...args) => chosen.push(args)} /> + </>)); + return { chosen, close: () => root.unmount() }; + }`, resolveDir: process.cwd(), loader: "tsx" }, + bundle: true, write: false, format: "iife", globalName: "AssertionMenuTest", platform: "browser", jsx: "automatic", + define: { "process.env.NODE_ENV": '"production"' }, + }); + const dom = new JSDOM('<html><body><div id="root"></div></body></html>', { + pretendToBeVisual: true, runScripts: "outside-only", url: "https://mobile-dev.test/", + }); + const window = dom.window; + window.ResizeObserver = class { observe() {} disconnect() {} unobserve() {} }; + Object.defineProperty(window.HTMLElement.prototype, "getBoundingClientRect", { value() { + return { x: 0, y: 0, left: 0, right: 200, top: 0, bottom: 40, width: 200, height: 40, toJSON() {} }; + } }); + window.eval(built.outputFiles[0].text); + const mounted = window.AssertionMenuTest.mount(); + t.after(() => { mounted.close(); window.close(); }); + const settle = () => new Promise(resolve => window.setTimeout(resolve, 25)); + const row = (kind: string) => window.document.querySelector<HTMLButtonElement>(`[data-recorder-offer="${kind}"]`)!; + const panel = (kind: string) => window.document.querySelector<HTMLElement>(`[data-recorder-assertion-panel="${kind}"]`); + + row("assertVisible").click(); + await settle(); + assert.deepEqual(Array.from(mounted.chosen[0]), ["assertVisible"]); + + row("assertVisible").dispatchEvent(new window.MouseEvent("mouseover", { bubbles: true })); + await new Promise(resolve => window.setTimeout(resolve, 120)); + assert.ok(panel("assertVisible"), "Hover opens the assertion choices"); + row("assertVisible").dispatchEvent(new window.MouseEvent("mouseout", { bubbles: true })); + await new Promise(resolve => window.setTimeout(resolve, 170)); + + window.document.querySelector<HTMLButtonElement>('[aria-label="Is Visible options"]')!.click(); + await settle(); + assert.ok(panel("assertVisible")); + assert.ok(panel("assertVisible")!.textContent?.includes("Insert")); + assert.equal(panel("assertVisible")!.textContent?.includes("Use default timeout"), false); + panel("assertVisible")!.querySelector<HTMLButtonElement>('[data-recorder-assertion-timeout]')!.click(); + await settle(); + const timeout = panel("assertVisible")!.querySelector<HTMLInputElement>('[aria-label="Assertion timeout in milliseconds"]')!; + assert.equal(timeout.value, "5000"); + timeout.value = "30000"; + panel("assertVisible")!.querySelector<HTMLButtonElement>('button[type="submit"]')!.click(); + await settle(); + assert.deepEqual(Array.from(mounted.chosen[1]), ["assertVisible", undefined, 30000]); + + row("assertText").click(); + await settle(); + assert.equal(mounted.chosen.length, 2); + assert.ok(panel("assertText")!.querySelector<HTMLInputElement>('[aria-label="Text to expect"]')); + panel("assertText")!.querySelector<HTMLButtonElement>('button[type="submit"]')!.click(); + await settle(); + assert.deepEqual(Array.from(mounted.chosen[2]), ["assertText", "Hello", undefined]); +}); diff --git a/packages/ui/test/code-highlight.test.ts b/packages/ui/test/code-highlight.test.ts new file mode 100644 index 00000000..d8970a07 --- /dev/null +++ b/packages/ui/test/code-highlight.test.ts @@ -0,0 +1,44 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { emitE2eTest } from "../../shared/src/test-recording.ts"; +import { changedLines, highlightCode } from "../src/code-highlight.ts"; + +test("highlighting never changes the code it renders", () => { + const source = emitE2eTest([ + { id: "1", kind: "tap", selector: { kind: "role", role: "tab", name: "Explore" }, label: "" }, + { id: "2", kind: "tap", selector: { kind: "point", point: { x: 10, y: 20.4 } }, label: "" }, + ], { title: 'a "quoted" flow' }); + assert.equal(highlightCode(source).map(token => token.text).join(""), source); +}); + +test("the kinds the preview colors are the ones the emitter writes", () => { + const tokens = highlightCode('// note\nawait screen.getByText("Save").tap();\nconst n = 42;'); + const of = (kind: string) => tokens.filter(token => token.kind === kind).map(token => token.text); + assert.deepEqual(of("comment"), ["// note"]); + assert.deepEqual(of("string"), ['"Save"']); + assert.deepEqual(of("keyword"), ["await", "const"]); + assert.deepEqual(of("call"), ["getByText", "tap"]); + assert.deepEqual(of("number"), ["42"]); + // `screen` is a plain identifier, not a call. + assert.ok(of("plain").includes("screen")); +}); + +test("a comment holding quotes or braces does not run on", () => { + const tokens = highlightCode('// TODO: add a testID to "Save"\nawait a();'); + assert.equal(tokens[0].kind, "comment"); + assert.equal(tokens[0].text, '// TODO: add a testID to "Save"'); + assert.ok(tokens.some(token => token.kind === "keyword" && token.text === "await")); +}); + +test("the preview can tell which lines a new step added", () => { + const before = ["a", "b", "});"]; + assert.deepEqual(changedLines(before, ["a", "b", "c", "});"]), [2, 3]); + // A step emits a TODO above its call, so a run of lines is as common as one. + assert.deepEqual(changedLines(before, ["a", "b", "// TODO", "c", "});"]), [2, 4]); + // Nothing moved, nothing is flagged. + assert.deepEqual(changedLines(before, before), [3, 3]); + // Removing a step flags nothing, since every remaining line was already there. + assert.deepEqual(changedLines(["a", "b", "c", "});"], before), [2, 2]); + // A changed line in the middle, such as the test title, is the only thing flagged. + assert.deepEqual(changedLines(["a", "b", "});"], ["a", "B", "});"]), [1, 2]); +}); diff --git a/packages/ui/test/popover-placement.test.ts b/packages/ui/test/popover-placement.test.ts new file mode 100644 index 00000000..eddb5211 --- /dev/null +++ b/packages/ui/test/popover-placement.test.ts @@ -0,0 +1,100 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { MARGIN, placeBelow, placeBeside } from "../src/popover-placement.ts"; +import type { Box } from "../src/popover-placement.ts"; + +/** A rectangle the way the DOM reports one. */ +const box = (left: number, top: number, width: number, height: number): Box => + ({ left, top, width, height, right: left + width, bottom: top + height }); + +const MENU = { width: 280, height: 230 }; +/** The command list: thirteen rows and two headings. */ +const COMMANDS = { width: 220, height: 520 }; + +test("a menu hangs below the element it is about when the window has room", () => { + const element = box(120, 300, 94, 54); + const placed = placeBelow(element, MENU, { width: 1200, height: 900 }); + assert.deepEqual({ left: placed.left, top: placed.top }, { left: 120, top: 360 }); +}); + +test("a menu with no room below it goes above the element rather than off the window", () => { + // The case the panel actually fails on: a tab at the bottom of a short panel. The old placement + // guessed a fixed 200px of menu and let the rest run past the bottom, where the stage clipped it. + const tab = box(240, 540, 94, 54); + const placed = placeBelow(tab, MENU, { width: 687, height: 640 }); + assert.equal(placed.top, 540 - 6 - 230, "above the element, its own height up from it"); + assert.ok(placed.top >= MARGIN && placed.top + MENU.height <= 640 - MARGIN, "wholly inside the window"); +}); + +test("a menu that fits neither way is pinned to the window instead of being cut off", () => { + const element = box(40, 150, 94, 54); + const placed = placeBelow(element, MENU, { width: 500, height: 300 }); + assert.equal(placed.top, 300 - MARGIN - 230); + assert.ok(placed.top >= MARGIN); + // Taller than the window: it scrolls, which is the only honest answer. + const tall = placeBelow(element, { width: 280, height: 1000 }, { width: 500, height: 300 }); + assert.equal(tall.maxHeight, 300 - MARGIN * 2); + assert.equal(tall.top, MARGIN); +}); + +test("a menu is pushed in from the edges it would cross", () => { + const view = { width: 400, height: 900 }; + assert.equal(placeBelow(box(380, 100, 20, 20), MENU, view).left, 400 - 280 - MARGIN); + assert.equal(placeBelow(box(-40, 100, 20, 20), MENU, view).left, MARGIN); +}); + +test("a submenu opens to the right, which is where its chevron points", () => { + const row = box(290, 500, 280, 36); + const roomy = placeBeside(row, COMMANDS, { width: 1200, height: 900 }); + assert.equal(roomy.left, 570 + 4, "beside the row it hangs off"); + assert.equal(roomy.top, 536 - 520, "bottom-aligned with the row, growing upward"); + + // The panel the recorder actually runs in is a few hundred points wide, so "flip when it does + // not fit" meant it opened on the left nearly always — a chevron pointing one way and a panel + // opening the other. It is pushed in from the window's edge instead. + const tight = placeBeside(row, COMMANDS, { width: 687, height: 640 }); + assert.equal(tight.left, 687 - 220 - MARGIN); + assert.ok(tight.left > row.left, "still to the right of the row, overlapping the menu it came from"); +}); + +test("a submenu taller than the room above its row is pushed down, not off the window", () => { + const row = box(100, 240, 280, 36); + const placed = placeBeside(row, COMMANDS, { width: 1200, height: 640 }); + assert.equal(placed.top, MARGIN, "its top stops at the window, and the rest of it scrolls"); + assert.equal(placed.maxHeight, 640 - MARGIN * 2); +}); + +test("a cascading branch flips to the left, but only when the left has room", () => { + const row = box(290, 500, 280, 36); + const panel = { width: 220, height: 200 }; + // Room on the right: the branch opens there, as its chevron says it will. + assert.equal(placeBeside(row, panel, { width: 1200, height: 900 }, { flip: true }).left, 574); + // No room on the right, room on the left: it flips rather than lying over the menu it came from, + // which by the third panel of a cascade is the trail the user is following. + assert.equal(placeBeside(row, panel, { width: 687, height: 640 }, { flip: true }).left, 290 - 4 - 220); + // Neither side fits, so it is pushed in from the edge — flipping here would point one way and + // open the other, which is the case `placeBeside` refuses by default. + const narrow = placeBeside(box(20, 500, 180, 36), panel, { width: 300, height: 640 }, { flip: true }); + assert.equal(narrow.left, 300 - 220 - MARGIN); + // Opting out is still the default, and still what the "All commands" submenu does. + assert.equal(placeBeside(row, panel, { width: 687, height: 640 }).left, 687 - 220 - MARGIN); +}); + +test("a branch beside a row in the middle of a list lines up with its top", () => { + const row = box(100, 240, 280, 36); + const panel = { width: 220, height: 200 }; + assert.equal(placeBeside(row, panel, { width: 1200, height: 900 }, { align: "top" }).top, 240); + assert.equal(placeBeside(row, panel, { width: 1200, height: 900 }).top, 276 - 200, "bottom is still the default"); +}); + +test("a branch asked for the left opens there even when the right has room", () => { + const row = box(600, 500, 240, 36); + const panel = { width: 230, height: 200 }; + // The step list is the rightmost pane in the Testing workspace, so its submenu opens over the + // preview rather than over the window's edge — whether or not the right would have fitted. + assert.equal(placeBeside(row, panel, { width: 1600, height: 900 }, { side: "left" }).left, 600 - 4 - 230); + // The default is unchanged: no `side`, and the right is where it goes. + assert.equal(placeBeside(row, panel, { width: 1600, height: 900 }).left, 840 + 4); + // With no room on the left either, the right is better than off the edge of the window. + assert.equal(placeBeside(box(10, 500, 120, 36), panel, { width: 1600, height: 900 }, { side: "left" }).left, 134); +}); diff --git a/packages/ui/test/react-ui.test.ts b/packages/ui/test/react-ui.test.ts index 8e8d08f6..67b2fa44 100644 --- a/packages/ui/test/react-ui.test.ts +++ b/packages/ui/test/react-ui.test.ts @@ -13,6 +13,7 @@ import { TraceSiftPanel } from "../src/tracesift-panel.ts"; import { AppIntelligencePanel } from "../src/app-intelligence-panel.ts"; import { RecordingController } from "../src/recording-controller.ts"; import type { AppFlowPanel } from "../src/app-flow-panel.ts"; +import { TestingPanel } from "../src/testing-panel.ts"; import { WorkspaceTools } from "../src/workspace-tools.ts"; import { recordingFixture } from "../../test-support/recording-fixtures.ts"; import type { App } from "@modelcontextprotocol/ext-apps"; @@ -126,7 +127,8 @@ test("React log controls filter virtual rows, attach full logs, and preserve sim show() {}, hide() {}, }; - const panels = new WorkspaceTools({ context: context as PanelContext, deviceApps, logs: panel, performance, storage, "app-intel": appIntel, tracesift, "app-flow": appFlow as unknown as AppFlowPanel }); + const testing = new TestingPanel(storageApp as unknown as App, deviceApps); + const panels = new WorkspaceTools({ context: context as PanelContext, deviceApps, logs: panel, performance, storage, "app-intel": appIntel, tracesift, "app-flow": appFlow as unknown as AppFlowPanel, testing }); deviceApps.selectDevice({ udid: UDID, name: "iPhone", state: "Booted", runtime: "iOS" }); performance.setAvailable(true); deviceApps.setAvailable(true); diff --git a/packages/ui/test/stored-test-steps.test.ts b/packages/ui/test/stored-test-steps.test.ts new file mode 100644 index 00000000..e389e4e9 --- /dev/null +++ b/packages/ui/test/stored-test-steps.test.ts @@ -0,0 +1,40 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { changeStoredStepCommand, duplicateStoredStep, moveStoredStep, removeStoredStep, storedStepCommands, storedTestCallLines, storedTestSteps, storedTestUntilSource } from "../src/stored-test-steps.ts"; + +test("saved tests show awaited calls at their source lines", () => { + const source = `import { test } from "@e2e-dev/mobile"; +test("flow", async ({ app, screen }) => { + await app.open(); + // await screen.tap(); is only a comment + await screen.getByRole("button", "Continue").tap(); + await expect(screen.getByText("Done")).toBeVisible(); +});`; + assert.deepEqual(storedTestSteps(source), [ + { line: 5, label: 'screen.getByRole("button", "Continue").tap()' }, + { line: 6, label: 'expect(screen.getByText("Done")).toBeVisible()' }, + ]); + assert.deepEqual(storedTestCallLines(source), [3, 5, 6], "live progress includes app.open and excludes comments"); +}); + +test("reorder, duplicate and command changes edit source without losing its wrapper", () => { + const source = 'import { test } from "@e2e-dev/mobile";\ntest("flow", async ({ app, screen }) => {\n await app.open();\n await screen.getByText("A").tap();\n await screen.getByText("B").tap();\n});'; + const moved = moveStoredStep(source, 4, 5); + assert.match(moved, /getByText\("B"\)\.tap\(\);\n await screen\.getByText\("A"\)\.tap\(\);/); + const duplicated = duplicateStoredStep(source, 4); + assert.equal((duplicated.match(/getByText\("A"\)\.tap/g) ?? []).length, 2); + const removed = removeStoredStep(source, 4); + assert.doesNotMatch(removed, /getByText\("A"\)/); + assert.match(removed, /getByText\("B"\)\.tap\(\);/); + const changed = changeStoredStepCommand(source, 4, "assertVisible"); + assert.match(changed, /import \{ expect \} from "e2e";/); + assert.match(changed, /await expect\(screen\.getByText\("A"\)\)\.toBeVisible\(\);/); + assert.ok(storedStepCommands(storedTestSteps(changed)[0]!).some(option => option.id === "tap")); + assert.ok(storedStepCommands({ line: 1, label: 'expect(screen.getByText("A")).toBeVisible({ timeout: 30000 })' }) + .some(option => option.id === "tap")); + assert.match(storedTestUntilSource(source, 4) ?? "", /await screen\.getByText\("A"\)\.tap\(\);\n return;\n await screen\.getByText\("B"\)\.tap\(\);/); + const withoutSemicolon = source.replace('getByText("A").tap();', 'getByText("A").tap()'); + assert.match(removeStoredStep(withoutSemicolon, 4), /getByText\("B"\)\.tap\(\);/); + assert.doesNotMatch(removeStoredStep(withoutSemicolon, 4), /getByText\("A"\)/); + assert.equal(storedTestUntilSource('await expect(\n screen.getByText("A")\n).toBeVisible();', 1), undefined); +}); diff --git a/packages/ui/test/test-recorder.test.ts b/packages/ui/test/test-recorder.test.ts new file mode 100644 index 00000000..1baacc22 --- /dev/null +++ b/packages/ui/test/test-recorder.test.ts @@ -0,0 +1,779 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { ScreenComponent } from "../../shared/src/screen-annotations.ts"; +import { TestRecorder, formatRunDuration } from "../src/test-recorder.ts"; +import { agentActSuggestion } from "../../shared/src/test-recording.ts"; + +const UDID = "11111111-2222-3333-4444-555555555555"; +const SCREEN = { width: 402, height: 874 }; + +function reader(components: ScreenComponent[], reads: { count: number; released?: number }) { + return { + readComponents: async () => { reads.count++; return components; }, + screen: () => SCREEN, + release: () => { reads.released = (reads.released ?? 0) + 1; }, + }; +} + +/** One element as the e2e runner reports it: its own role, and its accessible name in `label`. */ +const tab = (label: string, y: number): ScreenComponent => ({ + name: label, label, text: label, role: "button", depth: 1, source: "accessibility", nodeId: label, + bounds: { x: 0, y, width: 100, height: 40 }, +}); + +/** + * Records a step the way the panel does now: right-click the element, then pick a row. A left click + * records nothing at all, so there is no other way in. + */ +async function pick(recorder: TestRecorder, point: { x: number; y: number }, kind = "tap", value?: string) { + await recorder.openMenu(point); + await (recorder as unknown as { chooseOffer(kind: string, value?: string): Promise<void> }).chooseOffer(kind, value); +} + +test("a tap row records the step and takes it on the device", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + const done: unknown[] = []; + recorder.attach(UDID, "ios", { ...reader([tab("Explore", 800)], reads), perform: action => { done.push(action); } }); + recorder.start(UDID, "ios"); + + await pick(recorder, { x: 50, y: 820 }); + + const [step] = recorder.getSnapshot().steps; + assert.equal(step.kind, "tap"); + assert.deepEqual(step.selector, { kind: "role", role: "button", name: "Explore" }); + // A step that is recorded but never taken leaves the flow on the screen before it, so the row + // acts too — at the element's own centre, where the runner's locator action lands. + assert.deepEqual(done, [{ kind: "tap", point: { x: 50, y: 820 } }]); +}); + +test("an assertion is recorded from the element menu without touching the app", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + await recorder.openMenu({ x: 50, y: 820 }); + const offers = recorder.getSnapshot().menu?.offers ?? []; + // One row per kind of thing to do, not one per variant of the same selector. + assert.deepEqual(offers.map(offer => offer.kind), ["tap", "assertVisible", "scrollUntilVisible"]); + assert.equal(offers.find(offer => offer.kind === "assertVisible")?.source, + 'expect(screen.getByRole("button", "Explore")).toBeVisible()'); + await recorder.chooseOffer("assertVisible", undefined, 30_000); + + assert.equal(recorder.getSnapshot().menu, undefined, "choosing a row closes the menu"); + assert.equal(recorder.getSnapshot().steps[0].kind, "assertVisible"); + assert.equal(recorder.getSnapshot().steps[0].timeoutMs, 30_000); + assert.match(recorder.source, /toBeVisible\(\{ timeout: 30000 \}\)/); + assert.equal(reads.count, 1, "the menu and the step it records share one read"); +}); + +test("the tree is re-read after a tap but not after an assertion", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + await recorder.openMenu({ x: 50, y: 820 }); + await recorder.chooseOffer("assertVisible"); + assert.equal(reads.count, 1); + + // An assertion left the screen alone, so the cached tree still describes it. + await recorder.openMenu({ x: 50, y: 820 }); + await recorder.chooseOffer("assertVisible"); + assert.equal(reads.count, 1); + + // A tap can navigate, so the next step must be named from a fresh read. + await pick(recorder, { x: 50, y: 820 }); + assert.equal(reads.count, 1, "the tap itself is named from the screen it landed on"); + await pick(recorder, { x: 50, y: 820 }); + assert.equal(reads.count, 2, "the step after a tap re-reads the screen"); +}); + +test("nothing is recorded when no recording is running", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + + await pick(recorder, { x: 50, y: 820 }); + + assert.equal(recorder.getSnapshot().steps.length, 0); + assert.equal(reads.count, 0, "nothing is recording, so nothing is even read"); +}); + +test("a failed screen read drops the step and says why", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", { readComponents: async () => { throw new Error("device is gone"); }, screen: () => SCREEN }); + recorder.start(UDID, "ios"); + + await pick(recorder, { x: 50, y: 820 }); + + assert.equal(recorder.getSnapshot().steps.length, 0); + assert.match(recorder.getSnapshot().error, /device is gone/); +}); + +test("steps can be removed and the source re-emits from what is left", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800), tab("Home", 900)], reads)); + recorder.start(UDID, "ios"); + recorder.setTitle("two taps"); + + await pick(recorder, { x: 50, y: 820 }); + await pick(recorder, { x: 50, y: 920 }); + assert.match(recorder.source, /Explore/); + assert.match(recorder.source, /Home/); + + recorder.removeStep(recorder.getSnapshot().steps[0].id); + assert.doesNotMatch(recorder.source, /Explore/); + assert.match(recorder.source, /test\("two taps"/); +}); + +test("losing the device stops the recording", () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([], reads)); + recorder.start(UDID, "ios"); + assert.equal(recorder.isRecording(UDID, "ios"), true); + + recorder.detach(UDID, "ios"); + assert.equal(recorder.isRecording(UDID, "ios"), false); +}); + +test("hovering previews the element a tap would name", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + + // Nothing is outlined until recording starts: the mirror is a plain mirror the rest of the time. + await recorder.hover({ x: 50, y: 820 }); + assert.equal(recorder.getSnapshot().hover, undefined); + assert.equal(reads.count, 0, "hovering must not read the device when not recording"); + + recorder.start(UDID, "ios"); + await recorder.hover({ x: 50, y: 820 }); + const hover = recorder.getSnapshot().hover; + assert.equal(hover?.label, 'Tap "Explore" (button)'); + assert.deepEqual(hover?.bounds, { x: 0, y: 800, width: 100, height: 40 }); + assert.deepEqual(hover?.screen, SCREEN); + + // A warm tree answers further moves without another read. + await recorder.hover({ x: 60, y: 830 }); + assert.equal(reads.count, 1); + + await recorder.hover(); + assert.equal(recorder.getSnapshot().hover, undefined); +}); + +test("the outline is dropped when the screen it was resolved against is gone", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + await recorder.hover({ x: 50, y: 820 }); + assert.ok(recorder.getSnapshot().hover); + + await pick(recorder, { x: 50, y: 820 }); + assert.equal(recorder.getSnapshot().hover, undefined, "a tap changes the screen under the pointer"); + + await recorder.hover({ x: 50, y: 820 }); + assert.ok(recorder.getSnapshot().hover); + recorder.stop(); + assert.equal(recorder.getSnapshot().hover, undefined); +}); + +const CARD: ScreenComponent[] = [ + { name: "card", label: "Open city", identifier: "city-card", role: "button", depth: 1, source: "accessibility", nodeId: "card", bounds: { x: 0, y: 100, width: 300, height: 80 } }, + { name: "Paris", label: "Paris", text: "Paris", role: "text", depth: 2, source: "accessibility", nodeId: "label", parentId: "card", bounds: { x: 10, y: 110, width: 100, height: 20 } }, +]; + +test("the outline walks out to the enclosing element and records what it shows", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader(CARD, reads)); + recorder.start(UDID, "ios"); + + await recorder.hover({ x: 50, y: 120 }); + assert.equal(recorder.getSnapshot().hover?.label, 'Tap "Paris"'); + assert.equal(recorder.getSnapshot().hover?.levels, 2); + + recorder.walkHover(true); + assert.equal(recorder.getSnapshot().hover?.label, "Tap #city-card"); + assert.equal(recorder.getSnapshot().hover?.level, 1); + assert.equal(reads.count, 1, "walking re-resolves from the cache, it does not re-read the device"); + + // The step is what the outline showed, not what the pointer was over. + await pick(recorder, { x: 50, y: 120 }); + assert.deepEqual(recorder.getSnapshot().steps.at(-1)?.selector, { kind: "testId", testId: "city-card" }); + + // It cannot walk past the outermost element, nor inside the innermost. + await recorder.hover({ x: 50, y: 120 }); + recorder.walkHover(false); + assert.equal(recorder.getSnapshot().hover?.level, 0); + recorder.walkHover(false); + assert.equal(recorder.getSnapshot().hover?.level, 0); +}); + +test("an assertion on a point with nothing to name is refused, not recorded as comments", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + await recorder.openMenu({ x: 300, y: 200 }); + // A coordinate has nothing to be visible, so the menu offers the tap alone. + assert.deepEqual(recorder.getSnapshot().menu?.offers.map(offer => offer.kind), ["tap"]); + await recorder.chooseOffer("assertVisible"); + + assert.deepEqual(recorder.getSnapshot().steps, [], "nothing emittable, so nothing recorded"); + assert.match(recorder.getSnapshot().error, /nothing to act on but the coordinate/); +}); + +test("a coordinate tap carries a wait for the screen it was recorded on", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + await pick(recorder, { x: 300, y: 200 }); + + const step = recorder.getSnapshot().steps.at(-1); + assert.equal(step?.selector.kind, "point"); + assert.deepEqual(step?.anchor, { kind: "text", text: "Explore" }); + assert.match(recorder.source, /toBeVisible\(\);\n\s+await screen\.tapAt/); +}); + +test("recording again after a save starts a new flow rather than extending the saved one", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + recorder.stop(); + assert.equal(recorder.getSnapshot().steps.length, 1); + + // What the Testing panel passes when the steps on screen are already in a file. + recorder.start(UDID, "ios"); + assert.deepEqual(recorder.getSnapshot().steps, []); + + // And what it passes when they are not: stopping to scroll and carrying on keeps the flow. + await pick(recorder, { x: 50, y: 820 }); + recorder.stop(); + recorder.start(UDID, "ios", true); + assert.equal(recorder.getSnapshot().steps.length, 1); +}); + +test("the outline is drawn even where nothing can be named", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + await recorder.hover({ x: 300, y: 200 }); + + // Silence reads as a broken recorder; the mark says what a tap here would actually record. + const hover = recorder.getSnapshot().hover; + assert.match(hover?.label ?? "", /at 300, 200/); + assert.deepEqual(hover?.bounds, { x: 288, y: 188, width: 24, height: 24 }); +}); + +test("the device is given back on every path out of recording", async () => { + // Reading the runner's tree holds a session, and the runner allows one per device, so a session + // left open here makes the user's next `e2e run` fail with "device is already in use". + const reads = { count: 0, released: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + + recorder.stop(); + assert.equal(reads.released, 1, "stopping releases"); + + // Detaching with nothing recording still has a session to hand back. + recorder.detach(UDID, "ios"); + assert.equal(reads.released, 2, "detaching releases"); +}); + +test("detaching mid-recording stops and releases", async () => { + const reads = { count: 0, released: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + + recorder.detach(UDID, "ios"); + + assert.equal(recorder.isRecording(UDID, "ios"), false); + assert.equal(reads.released, 1, "the device must not be held by a recording that no longer exists"); +}); + +test("replacing the reader hands the old device back first", async () => { + const first = { count: 0, released: 0 }; + const second = { count: 0, released: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], first)); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], second)); + assert.equal(first.released, 1, "the reader being replaced was holding a session"); + assert.equal(second.released, 0); +}); + +test("recording selects the reader for that simulator instead of the last panel refreshed", async () => { + const recorder = new TestRecorder(); + const ios = { count: 0 }; + const android = { count: 0 }; + recorder.attach(UDID, "ios", reader([tab("iOS", 100)], ios)); + recorder.attach("emulator-5554", "android", reader([tab("Android", 100)], android)); + + recorder.start(UDID, "ios"); + await recorder.openMenu({ x: 50, y: 120 }); + + assert.equal(recorder.getSnapshot().menu?.target?.label, "iOS"); + assert.ok(ios.count > 0); + assert.equal(android.count, 0); + const androidReleases = android.released; + recorder.detach("emulator-5554", "android"); + assert.equal(android.released, androidReleases, "another panel disappearing cannot release the active recording"); + recorder.detach(UDID, "ios"); +}); + +test("a pointer move during the menu's own read does not drop the menu", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + let release = () => {}; + const pending = new Promise<void>(resolve => { release = resolve; }); + recorder.attach(UDID, "ios", { + readComponents: async () => { reads.count++; await pending; return [tab("Explore", 800)]; }, + screen: () => SCREEN, + release: () => undefined, + }); + recorder.start(UDID, "ios"); + + const opening = recorder.openMenu({ x: 50, y: 820 }); + // A browser sends a pointermove after the right-click. It used to bump the recorder's token and + // the menu, resolved against a token that no longer matched, was silently dropped. + await recorder.hover({ x: 51, y: 821 }); + release(); + await opening; + + assert.ok(recorder.getSnapshot().menu, "the menu survived the move that landed during its read"); + assert.equal(recorder.getSnapshot().menu?.offers[0]?.kind, "tap"); + recorder.detach(UDID, "ios"); +}); + +test("the verdict stays, the ticks go, and a failing mark stays with them gone", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], { count: 0 })); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + + recorder.beginRun(); + // A run starts with nothing marked: the runner says which step it is on, so nothing is guessed. + assert.deepEqual(recorder.getSnapshot().run?.lines, {}); + recorder.applyRunProgress([{ stepIndex: 1, phase: "start" }]); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 8: "running" }); + recorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }], durationMs: 8410 }); + assert.equal(recorder.getSnapshot().run?.fading, undefined, "the tick is held first, so it can be caught"); + + await new Promise(resolve => setTimeout(resolve, 2_700)); + assert.equal(recorder.getSnapshot().run?.fading, true); + await new Promise(resolve => setTimeout(resolve, 800)); + // The ticks go; the verdict and the duration are what the user came back for, and they stay. + assert.deepEqual(recorder.getSnapshot().run?.lines, {}, "the preview is the test, not a scoreboard"); + assert.equal(recorder.getSnapshot().run?.state, "passed"); + assert.equal(recorder.getSnapshot().run?.durationMs, 8410); + assert.equal(recorder.getSnapshot().run?.fading, undefined); + + // A mixed run: the ticks around the failure go, the failure itself does not. It is the only + // thing on screen that says *where*, so nothing times it out. + recorder.beginRun(); + recorder.finishRun({ status: "failed", durationMs: 420, steps: [ + { line: 8, status: "passed" }, { line: 9, status: "failed", error: "Timed out" }, + ] }); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 8: "passed", 9: "failed" }); + await new Promise(resolve => setTimeout(resolve, 3_500)); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 9: "failed" }); + assert.equal(recorder.getSnapshot().run?.state, "failed"); + assert.equal(recorder.getSnapshot().run?.durationMs, 420, "and how long it took before it broke"); + recorder.clearRun(); + recorder.detach(UDID, "ios"); +}); + +test("a run's marks go when the steps move out from under them, and the verdict is flagged", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], { count: 0 })); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + recorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }], durationMs: 8410 }); + + // The marks are claims about lines of the file as it was emitted, so the first edit to the steps + // moves the lines out from under them — well before the ticks would have faded on their own. + // The verdict and the duration are about a run that did happen, so they stay, flagged, so + // nothing reads as a verdict on what is on screen now. + await pick(recorder, { x: 50, y: 820 }); + const edited = recorder.getSnapshot().run; + assert.deepEqual(edited?.lines, {}, "no mark points at a line that has since changed"); + assert.equal(edited?.state, "passed"); + assert.equal(edited?.durationMs, 8410); + assert.equal(edited?.stale, true); + + // Every other way the steps move says the same thing, including a failing mark: it says where + // the run broke in a file that no longer has that line. + for (const edit of [ + () => recorder.duplicateStep(recorder.getSnapshot().steps[0]!.id), + () => recorder.moveStep(recorder.getSnapshot().steps[0]!.id, 1), + () => recorder.removeStep(recorder.getSnapshot().steps[0]!.id), + ]) { + recorder.finishRun({ status: "failed", steps: [{ line: 8, status: "failed", error: "Timed out" }] }); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 8: "failed" }); + edit(); + assert.deepEqual(recorder.getSnapshot().run?.lines, {}); + assert.equal(recorder.getSnapshot().run?.stale, true); + } + + // Starting a recording is the other way out: see `start`. + recorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }] }); + recorder.start(UDID, "ios", true); + assert.equal(recorder.getSnapshot().run, undefined); + recorder.detach(UDID, "ios"); +}); + +test("walking the outline out re-asks the menu about the element it walked to", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + const row = { + name: "Paris", label: "Paris", text: "Paris", role: "text", depth: 2, source: "accessibility" as const, + nodeId: "label", parentId: "card", bounds: { x: 20, y: 100, width: 80, height: 20 }, + }; + const card = { + name: "Open Paris", label: "Open Paris", role: "button", depth: 1, source: "accessibility" as const, + nodeId: "card", bounds: { x: 10, y: 90, width: 200, height: 60 }, + }; + recorder.attach(UDID, "ios", reader([card, row], reads)); + recorder.start(UDID, "ios"); + + await recorder.openMenu({ x: 50, y: 110 }); + assert.match(recorder.getSnapshot().menu!.offers[0].source, /getByText\("Paris"\)/, "the menu starts on the element under the pointer"); + + recorder.walkHover(true); + + assert.match(recorder.getSnapshot().menu!.offers[0].source, /getByRole\("button", "Open Paris"\)/, "and follows the outline out to the card"); + assert.deepEqual(recorder.getSnapshot().menu!.bounds, card.bounds, "including the rectangle it is anchored to"); + recorder.detach(UDID, "ios"); +}); + +test("re-attaching the same reader leaves the outline and the menu where they are", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + // The simulator panel re-attaches on every device-list refresh, with the same reader object. + // Treating that as a change threw away the cached tree and, through `clearHover`, took the + // outline and any open menu with it about a second after the user saw them. + const source = reader([tab("Explore", 800)], reads); + recorder.attach(UDID, "ios", source); + recorder.start(UDID, "ios"); + await recorder.openMenu({ x: 50, y: 820 }); + assert.ok(recorder.getSnapshot().menu); + assert.equal(reads.count, 1); + + recorder.attach(UDID, "ios", source); + + assert.ok(recorder.getSnapshot().menu, "a refresh is not a reason to close the menu"); + assert.ok(recorder.getSnapshot().hover, "nor to drop the outline"); + await recorder.hover({ x: 50, y: 821 }); + assert.equal(reads.count, 1, "nor to read the screen again"); + recorder.detach(UDID, "ios"); +}); + +test("live step events mark the line the step is on, by index not by guess", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800), tab("Home", 900)], { count: 0 })); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + await pick(recorder, { x: 50, y: 920 }); + + // `await app.open()` is a step the runner reports too, so index 0 is the header's line. + assert.deepEqual(recorder.emitted.calls, [6, 8, 9]); + + recorder.beginRun(); + assert.deepEqual(recorder.getSnapshot().run?.lines, {}, "nothing has started, so nothing is marked"); + + recorder.applyRunProgress([ + { stepIndex: 0, phase: "start" }, { stepIndex: 0, phase: "end", status: "passed" }, + { stepIndex: 1, phase: "start" }, + ]); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 6: "passed", 8: "running" }, + "the line the runner is on spins while the ones behind it are done"); + + recorder.applyRunProgress([ + { stepIndex: 0, phase: "start" }, { stepIndex: 0, phase: "end", status: "passed" }, + { stepIndex: 1, phase: "start" }, { stepIndex: 1, phase: "end", status: "failed", error: "Timed out" }, + ]); + assert.deepEqual(recorder.getSnapshot().run?.lines, { 6: "passed", 8: "failed" }); + assert.equal(recorder.getSnapshot().run?.error, "Timed out"); + + // An index past the file's awaited calls names no line rather than marking the wrong one. + recorder.applyRunProgress([{ stepIndex: 9, phase: "start" }]); + assert.deepEqual(recorder.getSnapshot().run?.lines, {}); + recorder.clearRun(); + recorder.detach(UDID, "ios"); +}); + +test("progress from a finished run is ignored, so the report has the last word", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], { count: 0 })); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + + recorder.beginRun(); + recorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }] }); + recorder.applyRunProgress([{ stepIndex: 1, phase: "start" }]); + + assert.deepEqual(recorder.getSnapshot().run?.lines, { 8: "passed" }, "a late poll cannot restart a finished run"); + recorder.clearRun(); + recorder.detach(UDID, "ios"); +}); + +test("the full command list is offered beside the short one, and records what it says", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + const done: unknown[] = []; + recorder.attach(UDID, "ios", { ...reader([tab("Explore", 800)], reads), perform: action => { done.push(action); } }); + recorder.start(UDID, "ios"); + + await recorder.openMenu({ x: 50, y: 820 }); + const commands = recorder.getSnapshot().menu?.commands ?? []; + // The short list stays short; everything else the runner can express is here. + assert.deepEqual(recorder.getSnapshot().menu?.offers.map(offer => offer.kind), ["tap", "assertVisible", "scrollUntilVisible"]); + assert.ok(commands.length > 3); + assert.equal(commands.find(offer => offer.kind === "assertNotVisible")?.source, + 'expect(screen.getByRole("button", "Explore")).not.toBeVisible()'); + + await recorder.chooseOffer("doubleTap"); + assert.equal(recorder.getSnapshot().steps[0].kind, "doubleTap"); + assert.match(recorder.source, /await screen\.getByRole\("button", "Explore"\)\.doubleTap\(\);/); + // The app has to end up on the screen the next step is resolved against, and the panel's input + // path has one tap in it, so the gesture is sent as two. + assert.deepEqual(done, [{ kind: "tap", point: { x: 50, y: 820 } }, { kind: "tap", point: { x: 50, y: 820 } }]); +}); + +test("a step can be duplicated, re-pointed at another command, and dragged into place", async () => { + const recorder = new TestRecorder(); + const field: ScreenComponent = { + name: "Search", label: "Search", role: "textbox", value: "hi", depth: 1, source: "accessibility", + nodeId: "search", bounds: { x: 0, y: 100, width: 100, height: 40 }, + }; + recorder.attach(UDID, "ios", reader([tab("Explore", 800), field], { count: 0 })); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + await pick(recorder, { x: 50, y: 120 }); + + const [explore, search] = recorder.getSnapshot().steps; + // What the element carried is kept with the step, so its own menu can offer the same commands + // the mirror's did long after that screen has gone. + assert.equal(search.target?.role, "textbox"); + assert.equal(search.target?.value, "hi"); + + recorder.duplicateStep(explore.id); + let steps = recorder.getSnapshot().steps; + assert.equal(steps.length, 3); + assert.notEqual(steps[2].id, explore.id, "a copy is its own step, not the same one twice"); + assert.deepEqual(steps[2].selector, explore.selector); + + // The command changes; the element it named does not. + recorder.changeStepCommand(steps[2].id, "assertNotVisible", undefined, 30_000); + steps = recorder.getSnapshot().steps; + assert.equal(steps[2].kind, "assertNotVisible"); + assert.equal(steps[2].label, 'Expect not visible "Explore" (button)'); + assert.deepEqual(steps[2].selector, explore.selector); + assert.match(recorder.source, /await expect\(screen\.getByRole\("button", "Explore"\)\)\.not\.toBeVisible\(\{ timeout: 30000 \}\);/); + + // A kind that carries text takes it; swapping back drops it rather than leaving it to emit. + recorder.changeStepCommand(steps[2].id, "assertText", "Explore"); + assert.equal(recorder.getSnapshot().steps[2].value, "Explore"); + recorder.changeStepCommand(steps[2].id, "assertVisible"); + assert.equal(recorder.getSnapshot().steps[2].value, undefined); + + // Reordering is the emitted order, so the file follows the list. + recorder.moveStep(steps[2].id, 0); + assert.deepEqual(recorder.getSnapshot().steps.map(step => step.id), [steps[2].id, explore.id, search.id]); + const lines = recorder.source.split("\n").filter(line => line.includes("await screen") || line.includes("await expect")); + assert.match(lines[0], /toBeVisible/); + assert.match(lines[1], /\.tap\(\)/); + + // Everything else about the list keeps working on top of the edits. + recorder.removeStep(explore.id); + assert.equal(recorder.getSnapshot().steps.length, 2); + recorder.clearSteps(); + assert.equal(recorder.getSnapshot().steps.length, 0); + recorder.detach(UDID, "ios"); +}); + +test("an agentic step is recorded from a sentence, even where nothing can be named", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + const done: unknown[] = []; + recorder.attach(UDID, "ios", { ...reader([tab("Explore", 800)], reads), perform: action => { done.push(action); } }); + recorder.start(UDID, "ios"); + + // The menu over an element carries what it is, so the Act row can draft a sentence out of it. + await recorder.openMenu({ x: 50, y: 820 }); + assert.equal(recorder.getSnapshot().menu?.target?.label, "Explore"); + assert.equal(agentActSuggestion(recorder.getSnapshot().menu?.target), 'tap the "Explore" button'); + + recorder.recordAgentStep({ kind: "agentAct", instruction: ' tap the "Explore" tab ' }); + const [act] = recorder.getSnapshot().steps; + assert.equal(act.kind, "agentAct"); + assert.equal(act.instruction, 'tap the "Explore" tab', "the sentence is trimmed, not the element's"); + assert.equal(act.selector, undefined, "an agent step names no element"); + assert.equal(act.label, 'Agent "tap the \\"Explore\\" tab"'); + // Nothing is sent to the device: running the instruction would be a run, not a recording. + assert.deepEqual(done, []); + // Opening the menu is what closes it; recording the step closes it too. + assert.equal(recorder.getSnapshot().menu, undefined); + + // Empty space resolves to a coordinate, which the pinned rows refuse — and which is exactly + // where describing the step is the only way to write it. + await recorder.openMenu({ x: 300, y: 200 }); + recorder.recordAgentStep({ kind: "agentAssert", instruction: "the city list is showing", vision: "only" }); + const assertStep = recorder.getSnapshot().steps.at(-1); + assert.equal(assertStep?.kind, "agentAssert"); + assert.equal(assertStep?.vision, "only"); + assert.equal(recorder.getSnapshot().error, "", "nothing refused the step"); + + // A blank sentence is not a step. + recorder.recordAgentStep({ kind: "agentAct", instruction: " " }); + assert.equal(recorder.getSnapshot().steps.length, 2); + + // Params ride on the act step and nothing else; no params means no options object at all. + recorder.recordAgentStep({ kind: "agentAct", instruction: "sign in", params: { email: "a@b.com", attempts: "3" } }); + const signIn = recorder.getSnapshot().steps.at(-1); + assert.deepEqual(signIn?.params, { email: "a@b.com", attempts: "3" }); + assert.match(recorder.source, /await agent\.act\("sign in", \{ params: \{ email: "a@b\.com", attempts: 3 \} \}\);/); + recorder.removeStep(signIn!.id); + + const source = recorder.source; + assert.match(source, /async \(\{ agent, app, screen \}\)/); + assert.match(source, /await agent\.act\("tap the \\"Explore\\" tab"\);/); + assert.match(source, /await agent\.assert\("the city list is showing", \{ vision: "only" \}\);/); + + // The editing an agent step does support still works on it; what it cannot do is take a locator. + recorder.duplicateStep(act.id); + assert.equal(recorder.getSnapshot().steps.length, 3); + recorder.changeStepCommand(act.id, "assertVisible"); + assert.equal(recorder.getSnapshot().steps[0].kind, "agentAct", "no command swap invents a locator"); + recorder.moveStep(act.id, 2); + assert.equal(recorder.getSnapshot().steps.at(-1)?.id, act.id); + recorder.detach(UDID, "ios"); +}); + +test("a swipe is recorded and taken on the device, scoped to the element or to the viewport", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + const done: any[] = []; + recorder.attach(UDID, "ios", { ...reader([tab("Explore", 800)], reads), perform: action => { done.push(action); } }); + recorder.start(UDID, "ios"); + + // Scoped: the gesture runs inside the element's own rectangle, well in from its edges, since a + // gesture begun on an edge is an edge gesture to the OS rather than the swipe the step means. + await pick(recorder, { x: 50, y: 820 }, "swipe", "left"); + const scoped = recorder.getSnapshot().steps.at(-1); + assert.equal(scoped?.kind, "swipe"); + assert.equal(scoped?.direction, "left"); + assert.equal(scoped?.label, 'Swipe left on "Explore" (button)'); + assert.deepEqual(done.at(-1), { kind: "swipe", from: { x: 85, y: 820 }, to: { x: 15, y: 820 } }); + assert.match(recorder.source, /await screen\.getByRole\("button", "Explore"\)\.swipe\(\{ direction: "left" \}\);/); + + // The viewport form, on a point nothing can name — the refusal every other non-tap command + // still gets must let this one through, because it is not about an element at all. + await pick(recorder, { x: 300, y: 200 }, "swipe", "up"); + const viewport = recorder.getSnapshot().steps.at(-1); + assert.equal(viewport?.selector.kind, "point"); + assert.equal(viewport?.direction, "up"); + assert.equal(recorder.getSnapshot().error, "", "nothing refused it"); + // The middle 70% of the whole screen, since there is no element to scope it to. + const { from, to } = done.at(-1); + assert.equal(from.x, SCREEN.width / 2); + assert.ok(Math.abs(from.y - (SCREEN.height / 2 + SCREEN.height * 0.35)) < 0.01); + assert.ok(Math.abs(to.y - (SCREEN.height / 2 - SCREEN.height * 0.35)) < 0.01, "the finger travels upwards"); + assert.match(recorder.source, /await screen\.swipe\(\{ direction: "up" \}\);/); + // Still a coordinate's screen, so it keeps the wait a point step needs. + assert.deepEqual(viewport?.anchor, { kind: "text", text: "Explore" }); + + // An unknown direction is not a reason to record a broken step; the menu's first one stands. + await pick(recorder, { x: 50, y: 820 }, "swipe", "sideways"); + assert.equal(recorder.getSnapshot().steps.at(-1)?.direction, "up"); + + // Swapping a swipe for a command that is not one drops the direction with it. + const swipe = recorder.getSnapshot().steps.at(-1)!; + recorder.changeStepCommand(swipe.id, "assertVisible"); + assert.equal(recorder.getSnapshot().steps.at(-1)?.direction, undefined); + recorder.detach(UDID, "ios"); +}); + +test("waitFor is recorded like the other judgements and emits the agent's own call", async () => { + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], { count: 0 })); + recorder.start(UDID, "ios"); + recorder.recordAgentStep({ kind: "agentWaitFor", instruction: "the spinner is gone" }); + recorder.recordAgentStep({ kind: "agentWaitFor", instruction: "the list has loaded", vision: "only" }); + const steps = recorder.getSnapshot().steps; + assert.deepEqual(steps.map(step => step.kind), ["agentWaitFor", "agentWaitFor"]); + assert.equal(steps[0].label, 'Agent wait "the spinner is gone"'); + assert.equal(steps[0].vision, undefined, "off is the runner's default, so it is not stored"); + assert.match(recorder.source, /await agent\.waitFor\("the spinner is gone"\);/); + assert.match(recorder.source, /await agent\.waitFor\("the list has loaded", \{ vision: "only" \}\);/); + assert.match(recorder.source, /async \(\{ agent, app, screen \}\)/); + recorder.detach(UDID, "ios"); +}); + +test("Record asks what to do with the steps already there, and either answer leaves a clean state", async () => { + const reads = { count: 0 }; + const recorder = new TestRecorder(); + recorder.attach(UDID, "ios", reader([tab("Explore", 800)], reads)); + recorder.start(UDID, "ios"); + await pick(recorder, { x: 50, y: 820 }); + recorder.stop(); + + // A finished run's marks are a verdict on the file as it was. + recorder.finishRun({ status: "passed", steps: [{ line: 8, status: "passed" }], durationMs: 8410 }); + assert.equal(recorder.getSnapshot().run?.state, "passed"); + assert.equal(recorder.getSnapshot().run?.durationMs, 8410); + + // Resume keeps the flow and adds to the end of it — this is the case that used to wipe, because + // saving clears the dirty flag and the old Record read that flag to decide. + recorder.start(UDID, "ios", true); + assert.equal(recorder.getSnapshot().steps.length, 1, "the recorded flow survives"); + assert.equal(recorder.getSnapshot().run, undefined, "and no stale overlay marks lines it no longer describes"); + await pick(recorder, { x: 50, y: 820 }); + assert.equal(recorder.getSnapshot().steps.length, 2, "the next step lands after the ones already there"); + + // New is the other answer, and it is now a thing the user asked for rather than a default. + recorder.finishRun({ status: "failed", steps: [], durationMs: 420 }); + recorder.start(UDID, "ios", false); + assert.deepEqual(recorder.getSnapshot().steps, []); + assert.equal(recorder.getSnapshot().run, undefined); + recorder.detach(UDID, "ios"); +}); + +test("a run's duration reads the way the runner's own CLI writes it", () => { + assert.equal(formatRunDuration(8410), "8.41s"); + assert.equal(formatRunDuration(1000), "1.00s"); + assert.equal(formatRunDuration(125_300), "125.30s"); + // Under a second it is milliseconds: `0.84s` reads as a rounding of something longer. + assert.equal(formatRunDuration(840), "840ms"); + assert.equal(formatRunDuration(999), "999ms"); + assert.equal(formatRunDuration(0), "0ms"); + assert.equal(formatRunDuration(-1), ""); + assert.equal(formatRunDuration(Number.NaN), ""); +}); + +test("recording an Android emulator reads the tree with its serial and platform", async () => { + const recorder = new TestRecorder(); + const seen: unknown[] = []; + recorder.useTreeSource({ read: async (deviceId, platform) => { seen.push([deviceId, platform]); return { tree: [tab("Home", 100)] }; }, release: () => undefined }); + recorder.attach("emulator-5554", "android", { readComponents: async () => (await recorder.readTree("emulator-5554", "android")).tree, screen: () => SCREEN }); + recorder.start("emulator-5554", "android"); + await new Promise(resolve => setTimeout(resolve, 20)); + assert.equal(recorder.isRecording("emulator-5554", "android"), true); + assert.equal(recorder.isRecording("emulator-5554", "ios"), false, "the same string on another platform is another device"); + assert.deepEqual(seen[0], ["emulator-5554", "android"]); + recorder.stop(); +}); diff --git a/packages/ui/test/test-source-suggestions.test.ts b/packages/ui/test/test-source-suggestions.test.ts new file mode 100644 index 00000000..173c4900 --- /dev/null +++ b/packages/ui/test/test-source-suggestions.test.ts @@ -0,0 +1,25 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { testSourceSuggestions } from "../src/test-source-suggestions.ts"; + +test("test source suggestions match the recorder's locator, action, and assertion API", () => { + const labels = (source: string) => testSourceSuggestions(source, source.length)?.items.map(item => item.label); + assert.deepEqual(labels("await exp"), ["expect visible", "expect not visible", "expect text"]); + assert.deepEqual(labels("await screen."), ["getByTestId", "getByRole", "getByText", "getByLabel", "getByPlaceholder"]); + assert.deepEqual(labels("await screen.getByT"), ["getByTestId", "getByText"]); + assert.deepEqual(labels('await screen.getByText("Save").ta'), ["tap"]); + assert.deepEqual(labels('await expect(screen.getByText("Save")).toB'), ["toBeVisible", "toBeEnabled", "toBeDisabled", "toBeSelected"]); + assert.deepEqual(labels('await expect(screen.getByText("Save")).'), + ["not", "toBeVisible", "toHaveText", "toContainText", "toHaveValue", "toBeEnabled", "toBeDisabled", "toBeSelected"]); + assert.deepEqual(labels('await expect(screen.getByText("Save")).not.'), + ["toBeVisible", "toHaveText", "toContainText", "toHaveValue", "toBeEnabled", "toBeDisabled", "toBeSelected"]); + assert.deepEqual(labels('await expect(screen.getByText("Save")).toBeVisible('), ["timeout"]); + assert.deepEqual(labels('await expect(screen.getByText("Save")).toHaveText("Done", '), ["timeout"]); + assert.deepEqual(labels('await agent.assert("The list is visible", '), + ["vision: true", 'vision: "only"', "vision: false"]); + assert.deepEqual(labels('await agent.assert("The list is visible", { vision: '), ["true", "false", '"only"']); + assert.deepEqual(labels('await agent.act("Sign in", '), ["params"]); + assert.deepEqual(labels("await app.op"), ["open"]); + assert.equal(labels("// screen."), undefined); + assert.equal(labels('await screen.getByText("screen.'), undefined); +}); diff --git a/packages/ui/test/testing-drafts-panel.test.ts b/packages/ui/test/testing-drafts-panel.test.ts new file mode 100644 index 00000000..995d3985 --- /dev/null +++ b/packages/ui/test/testing-drafts-panel.test.ts @@ -0,0 +1,370 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { HostBridge } from "../src/host-bridge.ts"; +import type { DeviceAppsStore } from "../src/device-apps.ts"; +import { TestingPanel } from "../src/testing-panel.ts"; +import { testRecorder } from "../src/test-recorder.ts"; +import type { ScreenComponent } from "../../shared/src/screen-annotations.ts"; + +const UDID = "11111111-2222-3333-4444-555555555555"; +const PROJECT = "/Users/someone/app"; +const DRAFT = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee"; + +const tab: ScreenComponent = { + name: "Explore", label: "Explore", role: "AXButton", depth: 1, source: "accessibility", nodeId: "explore", + bounds: { x: 0, y: 800, width: 100, height: 40 }, +}; + +const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => ({ device: undefined }) } as unknown as DeviceAppsStore; + +function host(responses: Record<string, unknown>, calls: { name: string; arguments: Record<string, unknown> }[]) { + return { + async callServerTool({ name, arguments: input }: { name: string; arguments: Record<string, unknown> }) { + calls.push({ name, arguments: input }); + return { content: [], structuredContent: responses[name] ?? {} }; + }, + } as unknown as HostBridge; +} + +function open(responses: Record<string, unknown>, calls: { name: string; arguments: Record<string, unknown> }[]) { + const panel = new TestingPanel(host(responses, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT }); + return panel; +} + +async function recordOneStep() { + testRecorder.attach(UDID, "ios", { readComponents: async () => [tab], screen: () => ({ width: 402, height: 874 }) }); + testRecorder.start(UDID, "ios"); + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("tap"); +} + +/** The panel's own autosave, without waiting out its debounce. */ +function keep(panel: TestingPanel): Promise<void> { + return (panel as unknown as { keepDraft(): Promise<void> }).keepDraft(); +} + +const named = (calls: { name: string }[], name: string) => calls.filter(call => call.name === name); + +test("the flow on screen is kept, so switching away does not lose it", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT, updatedAt: "2026-01-01T00:00:00.000Z" } }, calls); + await recordOneStep(); + + await keep(panel); + + const saved = named(calls, "mobile_testing_draft_save"); + assert.equal(saved.length, 1); + assert.equal(saved[0].arguments.projectPath, PROJECT); + assert.equal((saved[0].arguments.steps as unknown[]).length, 1); + assert.equal(saved[0].arguments.id, undefined, "the first keep creates the draft"); + assert.equal(panel.getSnapshot().draftId, DRAFT); + + // The next keep replaces the same draft rather than leaving a second copy of one flow. + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("assertVisible"); + await keep(panel); + const again = named(calls, "mobile_testing_draft_save"); + assert.equal(again.length, 2); + assert.equal(again[1].arguments.id, DRAFT); + panel.dispose(); +}); + +test("an unchanged flow is not rewritten, and an empty one is never kept", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT } }, calls); + testRecorder.stop(); + testRecorder.clearSteps(); + + await keep(panel); + assert.deepEqual(named(calls, "mobile_testing_draft_save"), [], "nothing to keep with no steps"); + + await recordOneStep(); + await keep(panel); + await keep(panel); + assert.equal(named(calls, "mobile_testing_draft_save").length, 1, "an unchanged flow is already kept"); + panel.dispose(); +}); + +test("opening a kept recording puts its steps back without arming the recorder", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const steps = [ + { id: "s1", kind: "tap", label: 'Tap "Explore" (tab)', selector: { kind: "role", role: "tab", name: "Explore" } }, + { id: "s2", kind: "assertVisible", label: 'Expect "Paris"', selector: { kind: "text", text: "Paris" } }, + ]; + const panel = open({ + mobile_testing_draft_save: { id: "11111111-1111-4111-8111-111111111111" }, + mobile_testing_draft_read: { draft: { id: DRAFT, title: "the explore screen", fileName: "explore", steps } }, + mobile_testing_drafts: { drafts: [] }, + }, calls); + await recordOneStep(); + + await panel.openDraft(DRAFT); + + assert.equal(testRecorder.getSnapshot().steps.length, 2); + assert.equal(testRecorder.getSnapshot().title, "the explore screen"); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, "", "reopening a flow must not start recording"); + assert.equal(panel.getSnapshot().draftId, DRAFT); + assert.equal(panel.getSnapshot().fileName, "explore"); + // The flow that was on screen is kept before it is replaced, so swapping drafts loses nothing. + assert.equal(named(calls, "mobile_testing_draft_save").length, 1); + // And the restored flow is not immediately written back as a change of its own. + await keep(panel); + assert.equal(named(calls, "mobile_testing_draft_save").length, 1); + panel.dispose(); +}); + +test("deleting the recording on screen clears the panel with it", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT }, mobile_testing_drafts: { drafts: [] } }, calls); + await recordOneStep(); + await keep(panel); + assert.equal(panel.getSnapshot().draftId, DRAFT); + + await panel.deleteDraft(DRAFT); + + assert.deepEqual(named(calls, "mobile_testing_draft_delete").map(call => call.arguments.id), [DRAFT]); + assert.equal(panel.getSnapshot().draftId, ""); + assert.equal(testRecorder.getSnapshot().steps.length, 0); + // And the cleared panel is not kept straight back as an empty draft. + await keep(panel); + assert.equal(named(calls, "mobile_testing_draft_save").length, 1); + panel.dispose(); +}); + +test("saving into the project forgets the draft it was standing in for", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ + mobile_testing_draft_save: { id: DRAFT }, + mobile_testing_drafts: { drafts: [] }, + mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` }, + }, calls); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ fileName: "flow" }); + await recordOneStep(); + await keep(panel); + + await panel.saveTest(); + await new Promise(resolve => setTimeout(resolve, 0)); + + assert.deepEqual(named(calls, "mobile_testing_draft_delete").map(call => call.arguments.id), [DRAFT]); + assert.equal(panel.getSnapshot().draftId, ""); + assert.equal(testRecorder.getSnapshot().steps.length, 1, "the steps stay on screen; only the kept copy goes"); + panel.dispose(); +}); + +test("nothing is kept while the agent is recording: the server holds that flow", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT } }, calls); + // After the panel's first agent poll, which reports no session and would otherwise clear this. + await new Promise(resolve => setTimeout(resolve, 0)); + await new Promise(resolve => setTimeout(resolve, 0)); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ + agent: { active: true, status: "recording", title: "a flow", goal: "explore", steps: 1, note: "", draftId: "" }, + }); + await recordOneStep(); + + await keep(panel); + + assert.deepEqual(named(calls, "mobile_testing_draft_save"), []); + panel.dispose(); +}); + +test("recording a second flow keeps a second draft rather than overwriting the first", async () => { + // The bug this covers: Record starts a new flow, the panel kept the finished one's id, and the + // next autosave wrote the new recording over the old one's file. Two recordings, one row. + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const ids = ["11111111-1111-4111-8111-111111111111", "22222222-2222-4222-8222-222222222222"]; + let next = 0; + const panel = new TestingPanel({ + async callServerTool({ name, arguments: input }: { name: string; arguments: Record<string, unknown> }) { + calls.push({ name, arguments: input }); + if (name === "mobile_testing_draft_save") return { content: [], structuredContent: { id: input.id ?? ids[next++] } }; + return { content: [], structuredContent: {} }; + }, + } as unknown as HostBridge, deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT }); + + await recordOneStep(); + await keep(panel); + assert.equal(panel.getSnapshot().draftId, ids[0]); + + // Record again, without resuming: a different recording, from the first step on. + testRecorder.start(UDID, "ios"); + assert.equal(panel.getSnapshot().draftId, "", "the finished flow's draft is let go as the new one begins"); + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("assertVisible"); + await keep(panel); + + const saved = named(calls, "mobile_testing_draft_save"); + assert.equal(saved.length, 2); + assert.equal(saved[1].arguments.id, undefined, "the second flow must not replace the first one's draft"); + assert.equal(panel.getSnapshot().draftId, ids[1]); + panel.dispose(); +}); + +test("resuming a flow goes on writing the same draft", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT } }, calls); + await recordOneStep(); + await keep(panel); + + // Record with `keepSteps`: the same recording, carried on. + testRecorder.start(UDID, "ios", true); + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("assertVisible"); + await keep(panel); + + const saved = named(calls, "mobile_testing_draft_save"); + assert.equal(saved.length, 2); + assert.equal(saved[1].arguments.id, DRAFT); + assert.equal((saved[1].arguments.steps as unknown[]).length, 2); + panel.dispose(); +}); + +test("Clear starts a flow of its own, so the cleared recording stays in the list", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ mobile_testing_draft_save: { id: DRAFT }, mobile_testing_drafts: { drafts: [] } }, calls); + await recordOneStep(); + await keep(panel); + + testRecorder.clearSteps(); + assert.equal(panel.getSnapshot().draftId, ""); + await recordOneStep(); + await keep(panel); + + const saved = named(calls, "mobile_testing_draft_save"); + assert.equal(saved.length, 2); + assert.equal(saved[1].arguments.id, undefined); + panel.dispose(); +}); + +test("a saved recording moves into the project's half of the list rather than leaving it", async () => { + // The bug this covers: the draft is deleted when the flow is written into the project, and + // nothing else listed the file, so pressing Save made the test vanish from the panel's list. + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ + mobile_testing_draft_save: { id: DRAFT }, + mobile_testing_drafts: { drafts: [] }, + mobile_testing_tests: { tests: [{ relativePath: "tests/flow.e2e.ts", name: "flow.e2e.ts", updatedAt: "2026-01-01T00:00:00.000Z", sizeBytes: 420 }] }, + mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` }, + }, calls); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ fileName: "flow" }); + await recordOneStep(); + await keep(panel); + + await panel.saveTest(); + await new Promise(resolve => setTimeout(resolve, 0)); + await new Promise(resolve => setTimeout(resolve, 0)); + + assert.ok(named(calls, "mobile_testing_tests").length > 0, "the project's tests are read again after a save"); + assert.deepEqual(panel.getSnapshot().tests.map(entry => entry.relativePath), ["tests/flow.e2e.ts"]); + assert.deepEqual(panel.getSnapshot().drafts, [], "and it is no longer one of the unstored recordings"); + panel.dispose(); +}); + +test("deleting one of the project's tests removes the file and re-reads the folder", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ + mobile_testing_tests: { tests: [] }, + mobile_testing_delete_file: { path: `${PROJECT}/tests/flow.e2e.ts`, removed: true }, + }, calls); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ + savedPath: `${PROJECT}/tests/flow.e2e.ts`, savedRevision: 3, + }); + + await panel.deleteTest("tests/flow.e2e.ts"); + + assert.deepEqual(named(calls, "mobile_testing_delete_file").map(call => call.arguments), + [{ projectPath: PROJECT, relativePath: "tests/flow.e2e.ts" }]); + assert.equal(panel.getSnapshot().deletingTest, "", "the row stops spinning when the delete lands"); + assert.equal(panel.getSnapshot().savedPath, "", "the \"Saved to …\" line is not left pointing at a file that is gone"); + assert.ok(named(calls, "mobile_testing_tests").length > 0); + panel.dispose(); +}); + +test("the project's tests are listed per project, and dropped when the project is left", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ + mobile_testing_tests: { tests: [{ relativePath: "tests/flow.e2e.ts", name: "flow.e2e.ts", updatedAt: "2026-01-01T00:00:00.000Z", sizeBytes: 10 }] }, + }, calls); + + await panel.refreshTests(); + assert.deepEqual(named(calls, "mobile_testing_tests").map(call => call.arguments), [{ projectPath: PROJECT }]); + assert.equal(panel.getSnapshot().tests.length, 1); + + // With no project there is nothing to list: another project's tests must not stay on screen. + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: "" }); + await panel.refreshTests(); + assert.deepEqual(panel.getSnapshot().tests, []); + panel.dispose(); +}); + +test("opening a project test previews its source and preserves the recording", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = open({ + mobile_testing_draft_save: { id: DRAFT }, + mobile_testing_read_file: { source: 'test("stored", async () => {});\n' }, + }, calls); + testRecorder.stop(); + testRecorder.clearSteps(); + await recordOneStep(); + const steps = testRecorder.getSnapshot().steps; + + await panel.openTest("tests/stored.e2e.ts"); + assert.equal(panel.getSnapshot().selectedTestPath, "tests/stored.e2e.ts"); + assert.match(panel.getSnapshot().selectedTestSource, /stored/); + assert.deepEqual(testRecorder.getSnapshot().steps, steps, "previewing leaves the editable flow intact"); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, "", "the mirror stops recording while a project file is previewed"); + assert.equal(named(calls, "mobile_testing_draft_save").length, 1, "the recording is kept before switching"); + assert.deepEqual(named(calls, "mobile_testing_read_file")[0].arguments, + { projectPath: PROJECT, relativePath: "tests/stored.e2e.ts" }); + + await panel.openDraft(DRAFT); + assert.equal(panel.getSnapshot().selectedTestPath, "", "reopening a draft exits the project preview"); + panel.dispose(); +}); + +test("project test edits stage before Save and Undo discards the working copy", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const source = 'test("saved", async () => {});'; + const panel = open({ + mobile_testing_read_file: { source, originalHash: "a".repeat(64), staged: false }, + mobile_testing_tests: { tests: [{ relativePath: "tests/saved.e2e.ts", name: "saved.e2e.ts", updatedAt: "2026-01-01T00:00:00.000Z", sizeBytes: 20 }] }, + mobile_testing_stage_file: { staged: true }, + mobile_testing_save_staged_file: { path: `${PROJECT}/tests/saved.e2e.ts`, originalHash: "b".repeat(64) }, + mobile_testing_discard_staged_file: { source, originalHash: "a".repeat(64) }, + }, calls); + await panel.openTest("tests/saved.e2e.ts"); + panel.editStoredTest("first edit"); + await panel.saveStoredTest(); + assert.deepEqual(named(calls, "mobile_testing_stage_file").map(call => call.arguments.source), ["first edit"]); + assert.equal(named(calls, "mobile_testing_save_staged_file").length, 1); + assert.equal(panel.getSnapshot().selectedTestStaged, false); + panel.editStoredTest("second edit"); + await panel.undoStoredTest(); + assert.equal(named(calls, "mobile_testing_stage_file")[1].arguments.originalHash, "b".repeat(64), "later edits use the newly saved file as their base"); + assert.equal(panel.getSnapshot().selectedTestSource, source); + assert.equal(panel.getSnapshot().selectedTestStaged, false); + assert.equal(named(calls, "mobile_testing_discard_staged_file").length, 1); + panel.dispose(); +}); + +test("Play until a saved call runs a temporary copy without saving the project file", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const source = 'test("flow", async ({ app, screen }) => {\n await app.open();\n await screen.getByText("A").tap();\n await screen.getByText("B").tap();\n});'; + const panel = open({ + mobile_testing_read_file: { source, originalHash: "a".repeat(64), staged: false }, + mobile_testing_run_test: { status: "passed", steps: [{ line: 3, status: "passed" }] }, + }, calls); + await panel.openTest("tests/flow.e2e.ts"); + await panel.runStoredTest(3); + const run = named(calls, "mobile_testing_run_test")[0]; + assert.ok(run); + assert.match(run.arguments.untilSource as string, /await screen\.getByText\("A"\)\.tap\(\);\n return;/); + assert.deepEqual(named(calls, "mobile_testing_save_staged_file"), []); + assert.deepEqual(named(calls, "mobile_testing_write_file"), []); + assert.equal(testRecorder.getSnapshot().run?.lines[3], "passed"); + panel.dispose(); +}); diff --git a/packages/ui/test/testing-panel.test.ts b/packages/ui/test/testing-panel.test.ts new file mode 100644 index 00000000..14c80545 --- /dev/null +++ b/packages/ui/test/testing-panel.test.ts @@ -0,0 +1,264 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { HostBridge } from "../src/host-bridge.ts"; +import type { DeviceAppsStore } from "../src/device-apps.ts"; +import { TestingPanel } from "../src/testing-panel.ts"; +import { OPEN_STEP, testRecorder } from "../src/test-recorder.ts"; +import type { ScreenComponent } from "../../shared/src/screen-annotations.ts"; + +const UDID = "11111111-2222-3333-4444-555555555555"; +const PROJECT = "/Users/someone/app"; + +const tab: ScreenComponent = { + name: "Explore", label: "Explore", role: "AXButton", depth: 1, source: "accessibility", nodeId: "explore", + bounds: { x: 0, y: 800, width: 100, height: 40 }, +}; + +// Saving touches no device, so the store is a stub: the real one binds to the document. +const deviceApps = { subscribe: () => () => undefined, getSnapshot: () => ({ device: undefined }) } as unknown as DeviceAppsStore; + +/** + * The calls the save and run path makes, with the panel's own background plumbing left out. + * + * An open panel also reports its selection to the server, polls for an agent recording — see + * `packages/core/src/testing-agent.ts` — and lists both halves of its own list: the recordings it + * has kept unsaved and the tests the project already holds. None of them is part of what saving or + * running does; they are asserted where they belong, in `testing-agent-panel.test.ts` and + * `testing-drafts-panel.test.ts`. + */ +const BACKGROUND = new Set([ + "mobile_testing_set_context", "mobile_testing_agent_state", "mobile_testing_projects", "mobile_testing_drafts", + "mobile_testing_tests", +]); +const flow = (calls: { name: string }[]) => calls.map(call => call.name).filter(name => !BACKGROUND.has(name)); + +function host(responses: Record<string, unknown>, calls: { name: string; arguments: Record<string, unknown> }[]) { + return { + async callServerTool({ name, arguments: input }: { name: string; arguments: Record<string, unknown> }) { + calls.push({ name, arguments: input }); + return { content: [], structuredContent: responses[name] ?? {} }; + }, + } as unknown as HostBridge; +} + +async function recordOneStep(panel: TestingPanel) { + testRecorder.attach(UDID, "ios", { readComponents: async () => [tab], screen: () => ({ width: 402, height: 874 }) }); + testRecorder.start(UDID, "ios"); + // A left click records nothing now; a step comes from the element menu. + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("tap"); + assert.equal(testRecorder.getSnapshot().steps.length, 1); + assert.equal(panel.dirty, true); +} + +test("a saved recording stops recording, so the next tap is not an unsaved step", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = new TestingPanel(host({ mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` } }, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + + await panel.saveTest(); + + assert.deepEqual(flow(calls), ["mobile_testing_write_file"]); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, "", "recording must end with the file it produced"); + assert.equal(panel.getSnapshot().savedPath, `${PROJECT}/tests/flow.e2e.ts`); + assert.equal(panel.dirty, false); + // The steps stay, so the user can add to the flow and save again — they just are not armed. + assert.equal(testRecorder.getSnapshot().steps.length, 1); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("a save blocked by an existing file leaves the recording running", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = new TestingPanel(host({ mobile_testing_write_file: { written: false, existed: true, path: `${PROJECT}/tests/flow.e2e.ts` } }, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + + await panel.saveTest(); + + assert.equal(panel.getSnapshot().overwritePath, `${PROJECT}/tests/flow.e2e.ts`); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, UDID, "nothing was written, so nothing is finished"); + testRecorder.stop(); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("Run saves, hands the device back, and only then runs", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = new TestingPanel(host({ + mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` }, + mobile_testing_run_test: { status: "passed", exitCode: 0, steps: [{ line: 8, status: "passed" }] }, + }, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + + await panel.runTest(); + + // The order is the feature: the recorder holds the one agent-device session the device allows, + // so a run spawned while it is held fails with "Device is already in use". + assert.deepEqual(flow(calls), + ["mobile_testing_write_file", "mobile_testing_release_device", "mobile_testing_run_test"]); + assert.equal(testRecorder.getSnapshot().recordingDeviceId, "", "a recording left armed would take the session straight back"); + assert.deepEqual(calls.findLast(call => call.name === "mobile_testing_run_test")?.arguments, + { projectPath: PROJECT, relativePath: "tests/flow.e2e.ts" }); + assert.equal(panel.getSnapshot().running, false); + assert.equal(testRecorder.getSnapshot().run?.state, "passed"); + assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 8: "passed" }); + testRecorder.clearRun(); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("a project test polls live progress against its own source lines", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + let finishRun!: (value: unknown) => void; + const pendingRun = new Promise(resolve => { finishRun = resolve; }); + const app = { + async callServerTool({ name, arguments: input }: { name: string; arguments: Record<string, unknown> }) { + calls.push({ name, arguments: input }); + if (name === "mobile_testing_run_test") { + const structuredContent = await pendingRun; + return { content: [], structuredContent }; + } + if (name === "mobile_testing_run_progress") return { content: [], structuredContent: { progress: [ + { stepIndex: 0, phase: "start" }, { stepIndex: 0, phase: "end", status: "passed" }, + { stepIndex: 1, phase: "start" }, + ] } }; + return { content: [], structuredContent: {} }; + }, + } as unknown as HostBridge; + const panel = new TestingPanel(app, deviceApps); + panel.setAvailable(true); + const source = 'test("stored", async ({ app, screen }) => {\n await app.open();\n await screen.getByText("Done").tap();\n});'; + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ + projectPath: PROJECT, selectedTestPath: "tests/stored.e2e.ts", selectedTestSource: source, configPath: `${PROJECT}/e2e.config.ts`, + }); + + const running = panel.runStoredTest(); + const deadline = Date.now() + 1_500; + while (!calls.some(call => call.name === "mobile_testing_run_progress") && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 25)); + } + + assert.ok(calls.some(call => call.name === "mobile_testing_run_progress"), "project runs poll while the runner is in flight"); + assert.deepEqual(testRecorder.getSnapshot().run?.lines, { 2: "passed", 3: "running" }, + "runner indexes map to the awaited calls in the saved project's source"); + finishRun({ status: "passed", exitCode: 0, steps: [{ line: 2, status: "passed" }, { line: 3, status: "passed" }] }); + await running; + + testRecorder.clearRun(); + panel.dispose(); +}); + +test("Play until here runs the flow cut short, and never writes the test", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = new TestingPanel(host({ + mobile_testing_run_test: { status: "passed", exitCode: 0, steps: [{ line: 8, status: "passed" }] }, + }, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + // A second step, so there is something below the one being played to. + await testRecorder.openMenu({ x: 50, y: 820 }); + await testRecorder.chooseOffer("tap"); + const [first] = testRecorder.getSnapshot().steps; + const full = testRecorder.source.split("\n"); + + await panel.runTest({ untilStepId: first!.id }); + + // No write at all: a half-run must not raise the overwrite question, and must not leave the + // saved test cut down to the part that was tried. + assert.deepEqual(flow(calls), ["mobile_testing_release_device", "mobile_testing_run_test"]); + assert.equal(panel.dirty, true, "nothing was saved, so the recording is still ahead of the file"); + const sent = calls.findLast(call => call.name === "mobile_testing_run_test")?.arguments ?? {}; + assert.equal(sent.relativePath, "tests/flow.e2e.ts"); + // The copy is the file's own text with the second step's line taken out and nothing else moved: + // every line it keeps is at the line number it has in the full file, which is what lets the + // runner's report mark the preview's own lines with no translation. + assert.deepEqual(String(sent.untilSource).split("\n"), [...full.slice(0, 8), ...full.slice(9)]); + assert.equal(testRecorder.getSnapshot().run?.state, "passed"); + assert.equal(testRecorder.getSnapshot().run?.untilLine, 8, "a verdict about part of the flow has to say so"); + + // The app opening on its own: the header's own call, with none of the steps under it. + await panel.runTest({ untilStepId: OPEN_STEP }); + const opening = String(calls.findLast(call => call.name === "mobile_testing_run_test")?.arguments.untilSource); + assert.equal(opening.includes("await app.open();"), true); + assert.deepEqual(opening.split("\n").filter(line => line.includes("await ")), [" await app.open();"], + "nothing is driven after it, which is the whole of what this run asks"); + assert.equal(testRecorder.getSnapshot().run?.untilLine, 6); + testRecorder.clearRun(); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("Run never replaces a file without being asked, and resumes from the answer", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const responses: Record<string, unknown> = { + mobile_testing_write_file: { written: false, existed: true, path: `${PROJECT}/tests/flow.e2e.ts` }, + mobile_testing_run_test: { status: "passed", exitCode: 0, steps: [{ line: 8, status: "passed" }] }, + }; + const panel = new TestingPanel(host(responses, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + + await panel.runTest(); + + assert.equal(panel.getSnapshot().overwritePath, `${PROJECT}/tests/flow.e2e.ts`); + assert.deepEqual(flow(calls), ["mobile_testing_write_file"], "nothing ran over a file the user has not agreed to replace"); + + responses.mobile_testing_write_file = { written: true, existed: true, path: `${PROJECT}/tests/flow.e2e.ts` }; + await panel.saveTest(true); + // Saving does not block on a run that may take minutes, so the resumed run lands after it. + await new Promise(resolve => setTimeout(resolve, 0)); + await new Promise(resolve => setTimeout(resolve, 0)); + // The answer finishes what Run began, so the user does not press Run twice. + assert.deepEqual(flow(calls), + ["mobile_testing_write_file", "mobile_testing_write_file", "mobile_testing_release_device", "mobile_testing_run_test"]); + testRecorder.clearRun(); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("a failing run keeps its mark on the line the report blames", async () => { + const calls: { name: string; arguments: Record<string, unknown> }[] = []; + const panel = new TestingPanel(host({ + mobile_testing_write_file: { written: true, existed: false, path: `${PROJECT}/tests/flow.e2e.ts` }, + mobile_testing_run_test: { status: "failed", exitCode: 1, steps: [{ line: 8, status: "passed" }, { line: 9, status: "failed", error: "Timed out" }] }, + }, calls), deviceApps); + panel.setAvailable(true); + (panel as unknown as { update(fields: Record<string, unknown>): void }).update({ projectPath: PROJECT, fileName: "flow" }); + await recordOneStep(panel); + + await panel.runTest(); + + const run = testRecorder.getSnapshot().run; + assert.equal(run?.state, "failed"); + assert.deepEqual(run?.lines, { 8: "passed", 9: "failed" }); + assert.equal(run?.error, "Timed out"); + assert.equal(run?.stale, undefined, "the result is about the file as it was just emitted"); + testRecorder.clearRun(); + testRecorder.clearSteps(); + testRecorder.detach(UDID, "ios"); + panel.dispose(); +}); + +test("a booted Android emulator is recordable and a target of the wrong platform is reported", () => { + const emulator = { udid: "emulator-5554", name: "Pixel", state: "Booted", runtime: "", platform: "android", kind: "emulator" }; + const store = { subscribe: () => () => undefined, getSnapshot: () => ({ device: emulator }) } as unknown as DeviceAppsStore; + const panel = new TestingPanel(host({}, []), store); + const update = (panel as unknown as { update(fields: Record<string, unknown>): void }).update.bind(panel); + update({ targets: [{ name: "ios", platform: "ios", bundleId: "a" }, { name: "android", platform: "android", bundleId: "a" }], targetName: "ios" }); + assert.match(panel.targetMismatch, /is ios, but the mirror shows an Android device/); + update({ targetName: "android" }); + assert.equal(panel.targetMismatch, ""); +}); diff --git a/packages/ui/test/workspace-devices.test.ts b/packages/ui/test/workspace-devices.test.ts new file mode 100644 index 00000000..e0328a12 --- /dev/null +++ b/packages/ui/test/workspace-devices.test.ts @@ -0,0 +1,85 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { WorkspaceDevices } from "../src/workspace-devices.ts"; +import type { SimulatorDevice } from "../../shared/src/protocol.ts"; + +const IPHONE: SimulatorDevice = { + udid: "11111111-2222-3333-4444-555555555555", name: "iPhone 17 Pro", state: "Booted", + platform: "ios", kind: "simulator", +} as SimulatorDevice; + +/** + * A simulator panel as `WorkspaceDevices` uses one: what it is listing, and whether it has been + * asked to load. The real one is a few thousand lines around a canvas and a device stream. + */ +function panel(platform: "ios" | "android") { + const stub = { + platform, + selected: undefined as SimulatorDevice | undefined, + loads: 0, + available: false, + openingTarget: undefined as unknown, + root: { hidden: true } as HTMLElement, + load: async () => { stub.loads++; }, + setAvailable: (value: boolean) => { stub.available = value; }, + setOpeningTarget: (target: unknown) => { stub.openingTarget = target; }, + }; + return stub; +} + +function workspace() { + const panels = [panel("ios"), panel("android")] as const; + const layouts: string[] = []; + const devices = new WorkspaceDevices(panels as never, layout => layouts.push(layout)); + return { devices, panels, layouts }; +} + +test("a device discovery finishes a layout the first device list left unresolved", async () => { + // The bug this covers: the opening tool starts the backend, so the first device list often comes + // back before anything has been found. That resolved the layout to "none" once and for good — + // both panels hidden, the platform selector reading "Platforms", and the panel telling the user + // to boot the simulator they already had booted. + const { devices, panels, layouts } = workspace(); + await devices.open({}); + await devices.setAvailable(true); + assert.deepEqual(layouts, ["none"], "nothing was found yet, so nothing is shown yet"); + + // Discovery finds the booted simulator a moment later and the panel selects it, which is the + // selection change the workspace passes on. + panels[0].selected = IPHONE; + devices.settleLayout(); + + assert.deepEqual(layouts, ["none", "ios"], "the mirror comes up on the platform that has a device"); +}); + +test("a resolved layout and a layout the user chose are both left alone", async () => { + const resolved = workspace(); + resolved.panels[0].selected = IPHONE; + await resolved.devices.open({}); + await resolved.devices.setAvailable(true); + assert.deepEqual(resolved.layouts, ["ios"]); + // A second device appearing does not move a workspace that is already showing one. + resolved.panels[1].selected = { ...IPHONE, platform: "android", kind: "emulator" } as SimulatorDevice; + resolved.devices.settleLayout(); + assert.deepEqual(resolved.layouts, ["ios"], "an answered layout is not answered again"); + + const chosen = workspace(); + await chosen.devices.open({}); + await chosen.devices.setAvailable(true); + assert.deepEqual(chosen.layouts, ["none"]); + chosen.devices.selectLayout(); + chosen.panels[0].selected = IPHONE; + chosen.devices.settleLayout(); + assert.deepEqual(chosen.layouts, ["none"], "the user's own choice of platforms is never overridden"); +}); + +test("a target naming a platform still decides the layout before anything is listed", async () => { + const { devices, panels, layouts } = workspace(); + await devices.open({ platform: "android" }); + assert.deepEqual(layouts, ["android"], "the task's platform is known, so there is nothing to wait for"); + assert.deepEqual(panels.map(entry => entry.openingTarget), [{ platform: "android" }, { platform: "android" }]); + // And a device turning up on the other platform does not move it. + panels[0].selected = IPHONE; + devices.settleLayout(); + assert.deepEqual(layouts, ["android"]); +}); diff --git a/tooling/hosts/hosts.spec.ts b/tooling/hosts/hosts.spec.ts index 4772ea92..0fa3f41e 100644 --- a/tooling/hosts/hosts.spec.ts +++ b/tooling/hosts/hosts.spec.ts @@ -66,6 +66,62 @@ test.describe("MCP Apps hosts", () => { expect(pageErrors).toEqual([]); }); + test("the Testing panel puts the mirror, the preview and the steps side by side", async ({ openView, page, server }) => { + test.skip(hasPanel(server) === false, "This target's server renders no panel in the chat."); + // Recording is done by tapping the mirror and read back in the preview beside it, so the three + // panes are a row: a quarter for the mirror, which is tapped, and the rest for what is read. + // This asks for the row itself rather than for `split`, which is false outside the fullscreen + // workspace — the mirror then kept the 60% default meant for a tool stacked under it, and the + // preview and the steps were left sharing the height below it. + for (const displayMode of ["fullscreen", "inline"] as const) { + await page.setViewportSize({ width: 1280, height: 800 }); + const { view } = await openView("mobile_open_workspace", { displayMode }); + // A mirror needs a platform to show: with none the pane is collapsed, which is the point of + // the shares below. These hosts see no devices, so the platform is chosen here. + await view.locator("#platform-select").click(); + await view.getByRole("option", { name: "iOS" }).click(); + await page.keyboard.press("Escape"); + await view.locator("#tool-select").click(); + await view.getByRole("option", { name: "Testing" }).click(); + await view.locator("[data-testing-recorder-workspace]").waitFor(); + const share = async (selector: string) => { + const box = await view.locator(selector).boundingBox(); + return Math.round((box?.width ?? 0) / 1280 * 100); + }; + // The panel's own padding and the gaps between the panes come off the two that are read, so + // the shares are 25/45/30 of the pane area and a little under that of the window. + expect(await share("#simulators-resizable"), `the mirror in ${displayMode}`).toBe(25); + expect(await share("#testing-preview-resizable"), `the preview in ${displayMode}`).toBeGreaterThan(40); + expect(await share("#testing-steps-resizable"), `the steps in ${displayMode}`).toBeGreaterThan(26); + // And in that order, left to right: the mirror leads, because it is what is acted on. + const mirror = await view.locator("#simulators-resizable").boundingBox(); + const preview = await view.locator("#testing-preview-resizable").boundingBox(); + const steps = await view.locator("#testing-steps-resizable").boundingBox(); + expect(mirror!.x).toBeLessThan(preview!.x); + expect(preview!.x).toBeLessThan(steps!.x); + } + }); + + test("a platform found while the Testing panel is open opens the mirror", async ({ openView, page, server }) => { + test.skip(hasPanel(server) === false, "This target's server renders no panel in the chat."); + // The ordinary order: the panel opens while the backend is still looking, and discovery + // selects a simulator a moment later. The mirror used to stay shut through that — the panel + // asked for its own shares during the commit that changed its panels' constraints, where the + // group discards them and keeps what it last requested, which was a mirror of nothing. + await page.setViewportSize({ width: 1280, height: 800 }); + const { view } = await openView("mobile_open_workspace", { displayMode: "fullscreen" }); + await view.locator("#tool-select").click(); + await view.getByRole("option", { name: "Testing" }).click(); + await view.locator("[data-testing-recorder-workspace]").waitFor(); + const mirror = view.locator("#simulators-resizable"); + // Nothing to mirror yet, so the pane is collapsed rather than an empty quarter of the window. + await expect.poll(async () => Math.round((await mirror.boundingBox())?.width ?? 0)).toBe(0); + await page.keyboard.press("Escape"); + await view.locator("#platform-select").click(); + await view.getByRole("option", { name: "iOS" }).click(); + await expect.poll(async () => Math.round((await mirror.boundingBox())?.width ?? 0)).toBe(320); + }); + test("Back to chat appears only where the host has no control of its own", async ({ openView, target, server }) => { test.skip(hasPanel(server) === false, "This target's server renders no panel in the chat."); const { view } = await openView("mobile_open_workspace", { displayMode: "fullscreen" }); diff --git a/tooling/hosts/screenshots/codex/workspace-menu-over-performance-dark.png b/tooling/hosts/screenshots/codex/workspace-menu-over-performance-dark.png index 3340a3e2..f314d5aa 100644 Binary files a/tooling/hosts/screenshots/codex/workspace-menu-over-performance-dark.png and b/tooling/hosts/screenshots/codex/workspace-menu-over-performance-dark.png differ diff --git a/tooling/plugin-hosts.mjs b/tooling/plugin-hosts.mjs index 2a11cf0f..68516845 100644 --- a/tooling/plugin-hosts.mjs +++ b/tooling/plugin-hosts.mjs @@ -32,7 +32,7 @@ export const pluginHosts = Object.fromEntries(readdirSync("apps") // Codex-only files would load the Codex launcher or manifest. export const codexOnlyPaths = [".mcp.json", "mcp.json", ".codex-plugin", "scripts/launch-mcp.sh"]; -export const sharedSkills = ["packages/skills/mobile-dev", "packages/skills/mobile-dev-setup", "packages/skills/agent-device"]; +export const sharedSkills = ["packages/skills/mobile-dev", "packages/skills/mobile-dev-setup", "packages/skills/mobile-dev-testing", "packages/skills/agent-device"]; export function pluginHost(name) { const host = pluginHosts[name]; diff --git a/website/astro.config.mjs b/website/astro.config.mjs index adc27f74..80d5f4de 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -25,6 +25,7 @@ const sidebar = [ { label: 'App control', slug: 'docs/app-control' }, { label: 'App Intelligence', slug: 'docs/app-intelligence' }, { label: 'App Flow', slug: 'docs/app-flow' }, + { label: 'Testing', slug: 'docs/testing' }, { label: 'Logs', slug: 'docs/logs' }, { label: 'Performance', slug: 'docs/performance' }, { label: 'Profiles', slug: 'docs/profiles' }, diff --git a/website/site-brief.md b/website/site-brief.md index 7939bb42..10c07e03 100644 --- a/website/site-brief.md +++ b/website/site-brief.md @@ -21,14 +21,14 @@ Status: approved homepage and 404, Starlight docs, social card, and the README b - **Device order:** iOS, Android; simulators/emulators, then physical devices. - **Frameworks:** Expo, React Native, SwiftUI, and other native mobile projects. - **Requirements:** Apple Silicon Mac; iOS simulator requires Xcode 26+, physical iOS discovery Xcode 27+; Android SDK/emulator or authorized device. -- **Features:** Live devices beside chat; human and agent device control; element and region annotations; app launch, permissions, deep links and device settings; native and Metro logs; CPU/memory and supported device Display FPS; TraceSift profile analysis; app storage inspection. +- **Features:** Live devices beside chat; human and agent device control; element and region annotations; app launch, permissions, deep links and device settings; App Intelligence and App Flow; manual and agent-driven e2e test recording on simulators and emulators; native and Metro logs; CPU/memory and supported device Display FPS; TraceSift profile analysis; app storage inspection. - **Limits:** Mobile Dev does not build the app itself; simulator Display FPS is unavailable; host capabilities affect panel presentation; physical iOS input and inspection have documented limits. - **Privacy:** Local device tooling; do not claim zero telemetry. Plugin Sentry reporting excludes screenshots, app logs, tool payloads, input content, profiles, storage contents and device performance recordings. Link to existing privacy and terms pages. - **Experimental:** No blanket experimental claim. Claude/Cursor distribution status is a manual preview, not a public marketplace install. - **FAQ:** Which environments? Which mobile frameworks? Physical devices? Requirements? Does this build my app? What telemetry is collected? Answers sourced from README and docs. - **Do not invent:** Adoption numbers, benchmark claims, testimonials, marketplace availability, cloud build services, universal feature parity between device kinds. - **Product vocabulary for scene design:** devices, streams, panels, shared screen, annotations, traces, logs, recordings, inspect, tap, type. -- **Source:** Current local README, CONTRIBUTING, docs/devices.md, docs/agent-workflow.md, product package manifests. +- **Source:** Current local README, CONTRIBUTING, docs/devices.md, docs/agent-workflow.md, docs/testing-panel.md, docs/agent-recording.md, and product package manifests. - **Links:** https://github.com/callstackincubator/mobile-dev ; support mailto:hello@callstack.com ; MIT license in the repository. Production domain not confirmed. - **Hosting / docs (confirmed):** Vercel; Astro with Starlight documentation, adapted from current user-facing docs. Leave contributor and implementation details in GitHub. - **Location (confirmed):** website/ inside the current repository. No existing website found. diff --git a/website/src/content/docs/docs/getting-started.md b/website/src/content/docs/docs/getting-started.md index 7b18c35c..de83bbbc 100644 --- a/website/src/content/docs/docs/getting-started.md +++ b/website/src/content/docs/docs/getting-started.md @@ -114,7 +114,7 @@ Follow your client's configuration format. Device panels and interactive recordi 1. Ask **“Build and run my app in the simulator beside this chat.”** You can name iOS, Android, a simulator model, or a connected device. 2. Choose or reuse a device. Selecting a stopped simulator or AVD boots it. -3. Watch the app beside the conversation. Use **Select** to annotate an element, or open **App Intelligence**, **Logs**, **Performance**, **Storage**, or **TraceSift**. +3. Watch the app beside the conversation. Use **Select** to annotate an element, or open **App Intelligence**, **Testing**, **Logs**, **Performance**, **Storage**, or **TraceSift**. Your app's own build tools compile, install, and launch it. Mobile Dev does not install Xcode runtimes or replace your project's build system. diff --git a/website/src/content/docs/docs/introduction.md b/website/src/content/docs/docs/introduction.md index 849a1862..19b59031 100644 --- a/website/src/content/docs/docs/introduction.md +++ b/website/src/content/docs/docs/introduction.md @@ -1,9 +1,9 @@ --- title: Introduction -description: Bring app structure, devices, logs, performance, and app storage into your ADE so you and your coding agent can work on the same mobile project. +description: Bring app structure, testing, devices, logs, performance, and app storage into your ADE so you and your coding agent can work on the same mobile project. --- -Mobile Dev is a plugin for mobile development in your agentic development environment (ADE). It brings your app's structure and iOS and Android devices beside your conversation, with tools to trace source dependencies, inspect the running app, read logs, record performance, and explore app storage. +Mobile Dev is a plugin for mobile development in your agentic development environment (ADE). It brings your app's structure and iOS and Android devices beside your conversation, with tools to trace source dependencies, record e2e tests, inspect the running app, read logs, record performance, and explore app storage. It works with Expo, React Native, SwiftUI, and other native mobile projects. Your project's own tools build and run the app. Mobile Dev supplies the device workspace and the information you and your agent use to work on it. @@ -23,6 +23,7 @@ The agent opens Mobile Dev, selects a suitable installed device, and uses the pr | Install or relaunch an app, open a deep link, change permissions, or test another language | [App control](/docs/app-control) | | Trace screens through their source files and packages without running the app | [App Intelligence](/docs/app-intelligence) | | Map an app's screens and navigation, capture each screen, and record a path to one | [App Flow](/docs/app-flow) | +| Record, edit, and run deterministic e2e tests from a simulator or emulator | [Testing](/docs/testing) | | Read native and Metro messages together and send a log to your agent | [Logs](/docs/logs) | | Record CPU, memory, and supported device Display FPS; select and compare ranges | [Performance](/docs/performance) | | Explore a CPU or React profile and investigate a measured finding | [Profiles](/docs/profiles) | diff --git a/website/src/content/docs/docs/privacy.md b/website/src/content/docs/docs/privacy.md index 28fd48a8..0033d0d9 100644 --- a/website/src/content/docs/docs/privacy.md +++ b/website/src/content/docs/docs/privacy.md @@ -42,6 +42,9 @@ Depending on the tools you use, Mobile Dev processes: - App Intelligence project paths, screen names and routes, project-relative source and asset paths, package names, static import relationships, and analysis diagnostics when you generate or query a source graph. +- Testing project paths and configuration, element trees, recorded steps, generated + test source, runner output, and locally kept unsaved recordings when you record, + edit, or run an e2e test. - App storage keys, values, database rows, and file metadata when you inspect storage. Results are filtered before reaching the host or panel: recognized sensitive fields, credential-like values, and binary values are redacted. These @@ -94,8 +97,8 @@ not a guarantee of anonymity. These IDs are excluded from performance metrics and span attributes. The reporting code excludes MCP arguments/results, screenshots, screen video, -app log content, typed/pasted text, App Intelligence graph contents, app storage -contents, profile contents, and device performance recordings. It scrubs common +app log content, typed/pasted text, App Intelligence graph contents, test source +and step contents, app storage contents, profile contents, and device performance recordings. It scrubs common secrets, email addresses, URLs, and local home paths from error text. Session Replay, minidumps, and profiling are disabled in the configured Sentry integrations. Scrubbing is a safeguard, @@ -183,6 +186,7 @@ and [OpenAI's privacy policy](https://openai.com/policies/privacy-policy/). | Mixpanel product analytics | Up to 24 months, then deleted. | | Local live logs | Held in bounded memory buffers. A server log session expires after five minutes without reads or keep-alives; closing the session stops collection. Copies already attached to the host follow the host's retention. | | Saved App Intelligence graphs and project history | Stored under `~/Library/Application Support/mobile-dev/app-intelligence` until you clear a graph or remove the files. Clearing a graph keeps the project in the recent-project list. | +| Testing project history and unsaved recordings | Stored under `~/Library/Application Support/mobile-dev/testing` until you delete a recording or remove the files. Saving a recording into the project removes the local draft it replaced; the draft store keeps at most 30 recordings. Tests saved into the project's `tests/` folder follow your repository's retention. | | Saved performance recordings | Stored under `~/Library/Application Support/mobile-dev/recordings` until you remove them; the recording store has no automatic age-based expiry. | | TraceSift integration results/references and the chosen project folder | Stored under `~/.tracesift/integrations/mobile-dev` by default, or the corresponding `TRACE_SIFT_HOME` directory, until removed. Original profiles and the TraceSift service's own data are separate. | | Installation ID | Stored at `~/Library/Application Support/mobile-dev/telemetry/anonymous-user-id` until you remove it. Process-session IDs last for the process; copies in sent reports follow Sentry retention. | @@ -192,8 +196,8 @@ and [OpenAI's privacy policy](https://openai.com/policies/privacy-policy/). Sentry's [data retention periods](https://docs.sentry.io/security-legal-pii/security/data-retention-periods/) apply to active telemetry data. Sentry separately [reports deleting production event-data backups](https://www.sentry.help/en/articles/13965019-how-frequently-is-data-backed-up) 30 or 90 days after backup creation, depending on the data type; that guidance does not specify an Application Metrics-specific backup period. Active-data expiry does not imply immediate removal from backups. -Uninstalling the plugin does not necessarily remove saved recordings, identifiers, -crash caches, TraceSift files, host messages, or backups. GitHub, the host, your +Uninstalling the plugin does not necessarily remove saved recordings, testing drafts, +identifiers, crash caches, TraceSift files, host messages, or backups. GitHub, the host, your operating system, and your own backup tools manage their copies separately. ## 6. Your choices and rights diff --git a/website/src/content/docs/docs/testing.md b/website/src/content/docs/docs/testing.md new file mode 100644 index 00000000..ad8f7cc1 --- /dev/null +++ b/website/src/content/docs/docs/testing.md @@ -0,0 +1,71 @@ +--- +title: Testing +description: Record, edit, and run deterministic e2e tests from an iOS simulator or Android emulator beside your conversation. +--- + +Open **Testing** to turn interactions with a running app into an editable e2e test. The mirror, generated TypeScript, and ordered step list share one workspace, so you can drive the app and review the test at the same time. + +Testing currently supports iOS simulators and Android emulators with an existing e2e project. Physical devices and Maestro output are not supported. Mobile Dev uses the project's own e2e CLI and `@e2e-dev/mobile` installation; it does not install or replace the project's test runner. + +## Prepare a project and device + +Before recording: + +1. Install the project's dependencies and configure an `e2e.config.ts`, `.mts`, `.js`, or `.mjs` file. +2. Boot an iOS simulator or Android emulator and select it in Mobile Dev. +3. Open **Testing** and choose the project folder. Mobile Dev remembers recent choices, but never infers a project from the server's working directory. +4. Choose a target whose platform matches the mirrored device, then select **Launch app**. + +The target needs a bundle ID in the e2e config so Mobile Dev can attach the recorder to the same app the test will open. If the selected target and device use different platforms, choose a matching target before recording. + +## Record a test + +Select **Record**. The mirror briefly shows that the recorder is preparing while it reads the element tree used by the project's own mobile test engine. + +While recording: + +- Click the mirror to use the app without adding a step. +- Right-click an element to open the commands it supports. +- Choose a tap, text entry, swipe, scroll, or assertion. Commands that need text ask for it before adding the step. +- Scroll over the outline to move from a nested element to the element around it. + +Mobile Dev prefers stable role, name, label, placeholder, and test-ID selectors. If several elements match, the generated locator records the selected position. A point that cannot be named can still produce a coordinate tap, but the preview marks that fallback so you can decide whether the app needs a stable test ID. + +The app performs action steps as they are recorded. Assertions only inspect the screen. Select **Stop** when the flow is complete; the steps and preview remain available. + +## Edit and organize steps + +The step list is the source of truth for a recording. You can: + +- reorder or duplicate a step; +- change a step to another command supported by the same element; +- remove individual steps or clear the flow after confirmation; +- edit the generated source directly when the structured step editor is not enough. + +Hovering or focusing a step highlights the lines it generated. The source preview also offers **Play until here** on executable lines, which runs a temporary shortened copy without changing the saved test. + +## Save and run + +Open **Test Description** to set the test title and file name. **Save** writes the test below the selected project's `tests/` directory. Mobile Dev refuses paths outside that directory and asks before replacing an existing file. + +**Run test** saves pending recording changes, releases the recorder's device session, and starts the project's local e2e CLI. The preview marks each reported step as it starts and finishes, then shows the runner's verdict and duration. A run never downloads a CLI with `npx`. + +Tests already in the project appear under **In the project**. Open one to preview its source, edit a temporary working copy, save or undo those edits, run the full file, or play to one step. Saving refuses to overwrite a file that changed on disk after you opened it. + +## Unsaved recordings + +Recordings that have not been written into the project appear under **Not stored**. Mobile Dev keeps them in its own application-support directory, scoped to the selected project, so switching tools or chats does not discard the flow. Saving moves the flow into the project's test list. Deleting an unsaved recording or a project test asks for confirmation because neither action can be undone in the panel. + +## Ask your agent to record + +You can ask your coding agent to record a screen or flow, for example: + +> Use Mobile Dev to record an e2e test for searching the city list. + +Choose the project, device, and target first. The agent opens Testing, reads the same runner-projected element tree, and adds one step at a time while the mirror shows the app changing. Manual input is locked during that session so two drivers cannot act on the same screen. + +When the agent finishes, its steps stay as an unsaved recording. The agent does not save or run the test; review the flow, adjust it, and choose when it should enter the repository. + +## Data and telemetry + +Test source, step contents, element names, labels, typed values, project paths, bundle IDs, and device identifiers are excluded from Mobile Dev telemetry. Aggregate timings and counts cover recorder preparation, element-tree reads, editing, runs, and agent-recording handoff. See [Privacy](/docs/privacy) for collection and opt-out details. diff --git a/website/src/pages/index.astro b/website/src/pages/index.astro index 5bc6535f..ede63e85 100644 --- a/website/src/pages/index.astro +++ b/website/src/pages/index.astro @@ -10,6 +10,7 @@ const tools = [ { name: 'Performance', href: docs.performance }, { name: 'Storage', href: docs.storage }, { name: 'App Intelligence', href: docs.appIntelligence }, + { name: 'Testing', href: docs.testing }, ]; --- diff --git a/website/src/site/content.ts b/website/src/site/content.ts index 6a9602b6..d5d1606e 100644 --- a/website/src/site/content.ts +++ b/website/src/site/content.ts @@ -13,8 +13,8 @@ export const site = { repo: 'https://github.com/callstackincubator/codex-mobile-dev-plugin', subtitle: 'Mobile development in your ADE', headline: ['Mobile development.', 'In your ADE.'], - lede: 'A plugin for mobile development in your ADE. Bring your app structure, running app, devices, logs, performance, and storage into Codex, Claude, Cursor, and more.', - description: 'A plugin for mobile development in your ADE. Run, inspect, and debug iOS and Android apps inside Codex, Claude, Cursor, and more.', + lede: 'A plugin for mobile development in your ADE. Bring your app structure, testing, running app, devices, logs, performance, and storage into Codex, Claude, Cursor, and more.', + description: 'A plugin for mobile development in your ADE. Build, test, inspect, and debug iOS and Android apps inside Codex, Claude, Cursor, and more.', install: 'codex plugin add mobile-dev@mobile-dev', skill: undefined as string | undefined, ogAlt: 'Mobile Dev by Callstack. Mobile development in your ADE. A coding conversation and a running phone app share one workspace.', @@ -37,6 +37,7 @@ export const docs = { control: at('/docs/app-control'), appIntelligence: at('/docs/app-intelligence'), appFlow: at('/docs/app-flow'), + testing: at('/docs/testing'), logs: at('/docs/logs'), performance: at('/docs/performance'), profiles: at('/docs/profiles'), @@ -96,6 +97,7 @@ export const features: Feature[] = [ { icon: 'flame', title: 'Profile analysis', body: 'Explore CPU and React profiles with TraceSift, and investigate selected frames with your agent.', href: docs.profiles }, { icon: 'network', title: 'App Intelligence', body: 'Trace each screen through your source files and packages without building or running the app.', href: docs.appIntelligence }, { icon: 'grid', title: 'App Flow', body: 'Map every screen and navigation edge, and capture each screen on a simulator or emulator.', href: docs.appFlow }, + { icon: 'test', title: 'Testing Studio', body: 'Record, edit, and run deterministic e2e tests from the app beside your conversation.', href: docs.testing }, { icon: 'database', title: 'App storage', body: 'Inspect files, databases, and key-value stores. Watch changes while you use your app.', href: docs.storage }, { icon: 'network', title: 'Your existing stack', body: 'Use your app’s own build tools with Expo, React Native, SwiftUI, and native projects.', href: docs.start }, ];