From ecbc64a24c09e205adb23c095a22c269dd4d2d65 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Tue, 6 Oct 2026 17:56:05 +0200 Subject: [PATCH 1/3] Add dragAndDrop tool for testing drag-and-drop scenarios --- CHANGELOG.md | 14 ++ docs/web-testing/agents.md | 2 +- docs/web-testing/page-interaction.md | 4 + src/action.ts | 24 +- src/ai/captain/web-mode.ts | 2 +- src/ai/driller.ts | 2 +- src/ai/pilot.ts | 2 +- src/ai/rerunner.ts | 3 +- src/ai/rules.ts | 22 ++ src/ai/tester.ts | 2 +- src/ai/tools.ts | 209 ++++++++++++++++++ src/utils/step-analyzer.ts | 4 +- test-data/drag-board.html | 67 ++++++ .../integration/drag-and-drop-browser.test.ts | 82 +++++++ tests/unit/drag-and-drop.test.ts | 197 +++++++++++++++++ tests/unit/executed-steps.test.ts | 12 + 16 files changed, 636 insertions(+), 12 deletions(-) create mode 100644 test-data/drag-board.html create mode 100644 tests/integration/drag-and-drop-browser.test.ts create mode 100644 tests/unit/drag-and-drop.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a14d99eb..a95f2779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 2026-10-06 + +### Changes +- [Tester] New `dragAndDrop()` tool, so scenarios that drag an item onto another item can be tested: reordering lists and trees, moving cards between columns, dropping items into folders or zones. The drag runs as a real mouse simulation first and, when the drop changes nothing, the same drag is retried with the native HTML5 drag and then with synthetic drag events — the last step covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags, and it is released even when the drag fails. Locators go to Playwright as written, so text is written as `text=...` — a bare string is read as CSS, and an object locator loses its text filter. + ```javascript + I.dragAndDrop('text=First chapter', 'text=Second chapter'); // move or reorder + ``` + Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records it as plain CodeceptJS — `I.pressKeyDown('Control')`, `I.dragAndDrop(...)`, `I.pressKeyUp('Control')` — so it replays and heals in rerun like any other step. +- Generated tests no longer break on a locator that contains quotes, such as `li:has-text("Item 2")`: executed commands are recorded with the quotes escaped, so the test file stays valid code and reruns instead of failing to load. + ```javascript + I.dragAndDrop('First chapter', 'Second chapter'); // move or reorder + ``` + Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records it as plain CodeceptJS — `I.pressKeyDown('Control')`, `I.dragAndDrop(...)`, `I.pressKeyUp('Control')` — so it replays and heals in rerun like any other step. + ## 2026-10-05 ### Changes diff --git a/docs/web-testing/agents.md b/docs/web-testing/agents.md index 82bbbe23..58bda6b7 100644 --- a/docs/web-testing/agents.md +++ b/docs/web-testing/agents.md @@ -16,7 +16,7 @@ flowchart LR ## Navigator Agent -Handles browser interactions: clicks, form fills, and navigation. +Handles browser interactions: clicks, drag and drop, form fills, and navigation. The Navigator runs CodeceptJS commands in the browser. When a selector fails, it tries other locator strategies and resolves the interaction without stopping the run. It remembers what worked and what didn't, so failed selectors don't keep killing your runs and tests survive UI changes. diff --git a/docs/web-testing/page-interaction.md b/docs/web-testing/page-interaction.md index 9c30663c..cad1d42b 100644 --- a/docs/web-testing/page-interaction.md +++ b/docs/web-testing/page-interaction.md @@ -72,6 +72,10 @@ The agent starts with short semantic locators from ARIA or visible text. When a When a locator fails, the agent tries the next strategy, then a visual click. Locators that worked are saved to experience and preferred on the next run. +## Drag and drop + +The Tester's `dragAndDrop()` tool drags one element onto another — reordering lists and trees, moving cards between columns, dropping items into folders or zones. It drags with a real mouse simulation first and, when the drop changes nothing, retries the same drag with the native HTML5 drag and then with synthetic drag events on its own — the last step covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags; it is recorded as plain CodeceptJS (`I.pressKeyDown`, `I.dragAndDrop`, `I.pressKeyUp`), so generated tests replay unchanged. + ## What happens after each action After every action, Explorbot captures the new page state and compares it with the previous one. The resulting diff tells the agent what changed — the URL, the accessibility tree, or the HTML — so it can confirm the action worked and decide what to do next. The Researcher turns a page into a structured UI map of sections and elements; see [Researcher](./researcher.md). diff --git a/src/action.ts b/src/action.ts index b13710fb..b948df15 100644 --- a/src/action.ts +++ b/src/action.ts @@ -588,11 +588,7 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra } if (batchFailed) return; - const executed: ExecutedStep = { command: step.toCode(), success: !error }; - if (Object.keys(step.opts || {}).length) { - const args = [step.humanizeArgs(), `step.opts(${JSON.stringify(step.opts)})`].filter(Boolean).join(', '); - executed.command = `${step.prefix}${step.actor}.${step.title}(${args})${step.suffix}`; - } + const executed: ExecutedStep = { command: stepCode(step), success: !error }; if (error) { executed.error = errorToString(error); batchFailed = true; @@ -617,6 +613,24 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra }; }; +function stepCode(step: any): string { + const needsEscaping = (step.args || []).some((arg: any) => typeof arg === 'string' && arg.includes('"')); + if (!needsEscaping) { + if (!Object.keys(step.opts || {}).length) return step.toCode(); + const args = [step.humanizeArgs(), `step.opts(${JSON.stringify(step.opts)})`].filter(Boolean).join(', '); + return `${step.prefix}${step.actor}.${step.title}(${args})${step.suffix}`; + } + const args = (step.args || []) + .filter((arg: any) => arg !== undefined) + .map((arg: any) => { + if (typeof arg === 'function') return arg.toString(); + if (arg && typeof arg.getMasked === 'function') return JSON.stringify(arg.getMasked()); + return JSON.stringify(arg); + }); + if (Object.keys(step.opts || {}).length) args.push(`step.opts(${JSON.stringify(step.opts)})`); + return `${step.prefix}${step.actor}.${step.name}(${args.join(', ')})${step.suffix}`; +} + const readFocusedElement = () => { const el = document.activeElement as any; if (!el || el === document.body) return null; diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index b58c0beb..24972862 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -135,7 +135,7 @@ export function WithWebMode(Base: T) { webModePrompt(): string { return dedent` - - Page actions: click, pressKey, form (CodeceptJS tools) + - Page actions: click, dragAndDrop, pressKey, form (CodeceptJS tools) - Navigation: navigate() — AI-powered navigation to URLs or page descriptions - Browser diagnostics: browser() — inspect status, evaluate JS, close tabs, screenshot, reload, recover closed/crashed pages, restart browser, open a fresh tab - Visual analysis: see() — screenshot-based page verification diff --git a/src/ai/driller.ts b/src/ai/driller.ts index b5c48c98..bbc95936 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -63,7 +63,7 @@ interface DrillOptions { } export class Driller extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'dragAndDrop', 'pressKey', 'form']; emoji = 'D'; private navigator: Navigator; private currentPlan?: Plan; diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 7d204bd4..fc130451 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1251,7 +1251,7 @@ export class Pilot implements Agent { To request more context, mention ATTACH_ARIA, ATTACH_SUMMARY, or ATTACH_UI_MAP — only when recent actions show failures. - Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck, + Tester tools: click, hover, dragAndDrop, pressKey, form, see, verify, interact, context, research, xpathCheck, visualClick, back, getVisitedStates, reset, stop, finish, record. Use tool names exactly as listed. Do not invent combined names or aliases. Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.refreshPage() through form. diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index 8e8762b0..d3d3a42c 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -31,7 +31,7 @@ import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts'; const debugLog = createDebug('explorbot:rerunner'); export class Rerunner extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'dragAndDrop', 'pressKey', 'form']; emoji = '🔄'; private agentTools: any; @@ -441,6 +441,7 @@ export class Rerunner extends TaskAgent implements Agent { - You MUST execute the replacement action — not just diagnose - Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element + - Use dragAndDrop() to drag an element onto another element, with a modifier for copy-drags - Use form() for text input, dropdown selection, file uploads - Use pressKey() for special keys or key combinations - Use wait() when page is loading — returns fresh ARIA automatically diff --git a/src/ai/rules.ts b/src/ai/rules.ts index d807f6b5..afcdee19 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -505,6 +505,28 @@ export const actionRule = dedent` After hovering, use see() or context() to check what appeared. + ### I.dragAndDrop + + Drags an element and drops it onto another element. Run via the dragAndDrop() tool, not form(). + Use for reorderable lists and trees, moving cards between columns, dropping items into folders or zones. + + I.dragAndDrop(, ) + + Unlike I.click, locators here go to Playwright as written: a bare string is read as CSS + and an object locator loses its text filter. Write text as 'text=...' and prefer CSS or XPath. + When an item drags only from a handle inside it, target the handle as the source. + After a result reported several matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument. + For a drag with a modifier held (a copy-drag that duplicates the item), pass the modifier to the dragAndDrop() tool. + + + I.dragAndDrop('text=First chapter', 'text=Second chapter') + I.dragAndDrop('.card-new', '.column-done') + + + The tool drags with a real mouse simulation first and retries with the native HTML5 drag + and then synthetic drag events when the drop changes nothing. When a command fails, the problem + is the locators — change locators, not the mechanism. + ### I.seeInClipboard / I.seeClipboardEquals / I.clearClipboard Check what a copy control put into the system clipboard. Run via form(), not click(). diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 3b752812..2f7bd2c9 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -40,7 +40,7 @@ const SAMPLE_FILES: Record = { }; export class Tester extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'hover', 'dragAndDrop', 'pressKey', 'form']; protected readonly DELEGATED_ACTION_TOOLS = ['interact', 'visualClick']; protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe']; emoji = '🧪'; diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 3fb3485a..3b563218 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -9,6 +9,7 @@ import { type Task, TestResult } from '../test-plan.js'; import { ariaRefSelector, describeRef, refIsGone } from '../utils/aria-ref.ts'; import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; +import { splitTopLevel } from '../utils/code-extractor.ts'; import { cleanHtmlSnippet } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; import { pause } from '../utils/loop.js'; @@ -257,6 +258,144 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep }, }), + dragAndDrop: tool({ + description: dedent` + Drag an element and drop it onto another element by trying multiple CodeceptJS commands in order. + + Use for any drag-and-drop interaction: reordering list or tree items, moving cards between + columns, dropping items into folders or zones. + + I.dragAndDrop(source, target) - drag the element matching the source locator onto the element + matching the target locator. + Unlike I.click, locators here go to Playwright as written: a bare string is read as CSS and + an object locator loses its text filter. Write text as 'text=...' and prefer CSS or XPath. + After a result reported several matches, reuse that locator with step.opts({ elementIndex: N }) + as the last argument. + + The tool drags with a real mouse simulation first and, when the drop changes nothing, retries + the same drag with the native HTML5 drag and then with synthetic drag events — you never choose + between them. When a command fails, the problem is the locators: change locators, not the mechanism. + For a drag with a modifier held (a copy-drag that duplicates the item), pass modifier. + + CRITICAL: All commands MUST drag the SAME element onto the SAME target. + This is a FALLBACK list, NOT a sequence of different drags. + `, + inputSchema: z.object({ + commands: z.array(z.string()).describe(dedent` + FALLBACK commands for ONE drag. All commands must drag the SAME source onto the SAME target. + Each command is a full I.dragAndDrop(source, target) with both locators, ordered by reliability: + 1. I.dragAndDrop('text=First chapter', 'text=Second chapter') - by visible text + 2. I.dragAndDrop(CSS, CSS) - e.g. I.dragAndDrop('.card-new', '.column-done') + 3. I.dragAndDrop(XPath, XPath) - when locators already include context + A bare string is read as CSS, and an object locator loses its text filter — never use them here. + After a result reporting multiple matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument. + `), + modifier: z.enum(['Control', 'Shift', 'Alt', 'Meta', 'CommandOrControl']).optional().describe('Modifier key held during the whole drag, e.g. Control for a copy-drag that duplicates the item.'), + explanation: z.string().describe('Why you are dragging this element'), + }), + execute: async ({ commands: rawCommands, modifier, explanation }) => { + const activeNote = task.startNote(explanation); + + if (rawCommands.length === 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'No commands provided'); + } + + const commands = rawCommands.map((cmd) => cmd.trim()); + const invalidCommands = commands.filter((cmd) => !cmd.startsWith('I.dragAndDrop')); + + if (invalidCommands.length > 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `Invalid commands: ${invalidCommands.join(', ')}. Each command must be a full I.dragAndDrop(source, target).`, { + suggestion: 'Use click() to click elements, or form() for other command sequences.', + }); + } + + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + + for (let i = 0; i < commands.length; i++) { + const command = commands[i]; + const block = withModifier(command, modifier); + const success = await action.attempt(block, explanation); + + const attempt: { command: string; success: boolean; error?: string } = { command: block, success }; + if (action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); + + if (!success) { + await releaseModifier(action, modifier); + continue; + } + + let executedCode = action.executedSteps.map((step) => step.command).join('\n') || block; + let toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, executedCode); + + if (!hasObservablePageChange(toolResult) && !command.includes('force')) { + const forceBlock = withModifier(`${command.replace(/\)\s*$/, '')}, { force: true })`, modifier); + const forceSuccess = await action.attempt(forceBlock, explanation); + + const forceAttempt: { command: string; success: boolean; error?: string } = { command: forceBlock, success: forceSuccess }; + if (!forceSuccess && action.lastError) forceAttempt.error = errorText(action.lastError); + attempts.push(forceAttempt); + + if (forceSuccess) { + executedCode = action.executedSteps.map((step) => step.command).join('\n') || forceBlock; + toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, executedCode); + } else { + await releaseModifier(action, modifier); + } + } + + if (!hasObservablePageChange(toolResult)) { + const dragged = await syntheticDragDrop(explorer, command, modifier); + attempts.push({ command: `${command} (synthetic drag events)`, success: dragged }); + + if (!dragged) { + await releaseModifier(action, modifier); + continue; + } + + const replayable = withModifier(command, modifier); + const syntheticResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, replayable); + + if (!hasObservablePageChange(syntheticResult)) { + await releaseModifier(action, modifier); + continue; + } + + executedCode = replayable; + toolResult = { ...syntheticResult, message: 'The drop landed only after synthetic drag events were dispatched; the recorded command replays through the tool, which runs the same chain again.' }; + } + + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + const data: Record = { ...toolResult, attempts, code: executedCode }; + const notExecuted = commands.slice(i + 1); + if (notExecuted.length) { + data.notExecuted = notExecuted; + data.suggestion = `SKIPPED: ${notExecuted.join('; ')}`; + } + return successToolResult('dragAndDrop', data, action); + } + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'dragAndDrop', + 'All dragAndDrop commands failed', + { + ...toolResult, + attempts, + suggestion: 'A failed drag is a locator problem, not a drag mechanism problem. Check with xpathCheck() that the source and the target each match exactly one element, and that the target is a valid drop zone.', + }, + action.lastError, + judge, + explanation + ); + }, + }), + pressKey: tool({ description: dedent` Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers. @@ -1243,6 +1382,76 @@ function errorText(error: unknown): string { return 'Unknown error occurred'; } +function withModifier(command: string, modifier?: string): string { + if (!modifier) return command; + return `I.pressKeyDown(${JSON.stringify(modifier)})\n${command}\nI.pressKeyUp(${JSON.stringify(modifier)})`; +} + +async function releaseModifier(action: any, modifier?: string): Promise { + if (!modifier) return; + await action.attempt(`I.pressKeyUp(${JSON.stringify(modifier)})`, `Release the ${modifier} key left held by a failed drag`); +} + +const SYNTHETIC_MODIFIER_KEYS: Record = { Control: 'ctrlKey', Shift: 'shiftKey', Alt: 'altKey', Meta: 'metaKey', CommandOrControl: 'auto' }; + +function unquoteCommandArg(arg: string): string { + if (arg.startsWith('"')) { + try { + return JSON.parse(arg); + } catch { + return arg; + } + } + return arg + .replace(/^['"`]/, '') + .replace(/['"`]$/, '') + .replace(/\\(['"`])/g, '$1'); +} + +function dragLocators(command: string): string[] { + const inner = command.replace(/^I\.dragAndDrop\(/, '').replace(/\)\s*$/, ''); + return splitTopLevel(inner, ',') + .map((part) => part.trim()) + .filter((part) => /^['"`]/.test(part)) + .slice(0, 2) + .map(unquoteCommandArg); +} + +export async function syntheticDragDrop(explorer: any, command: string, modifier?: string): Promise { + const [src, dst] = dragLocators(command); + if (!src || !dst) return false; + const modifierKey = modifier ? SYNTHETIC_MODIFIER_KEYS[modifier] : undefined; + return explorer + .withPage(async (page: any) => { + const [srcBox, dstBox] = await Promise.all([page.locator(src).boundingBox(), page.locator(dst).boundingBox()]); + if (!srcBox || !dstBox) return false; + const from = { x: srcBox.x + srcBox.width / 2, y: srcBox.y + srcBox.height / 2 }; + const to = { x: dstBox.x + dstBox.width / 2, y: dstBox.y + dstBox.height / 2 }; + return page.evaluate( + ({ from, to, modifierKey }) => { + const startEl = document.elementFromPoint(from.x, from.y); + const endEl = document.elementFromPoint(to.x, to.y); + if (!startEl || !endEl) return false; + const key = modifierKey === 'auto' ? (navigator.platform.includes('Mac') ? 'metaKey' : 'ctrlKey') : modifierKey; + const dataTransfer = new DataTransfer(); + const fire = (el: Element, type: string, at: { x: number; y: number }) => { + const init: any = { bubbles: true, cancelable: true, dataTransfer, clientX: at.x, clientY: at.y }; + if (key) init[key] = true; + el.dispatchEvent(new DragEvent(type, init)); + }; + fire(startEl, 'dragstart', from); + fire(endEl, 'dragenter', to); + fire(endEl, 'dragover', to); + fire(endEl, 'drop', to); + fire(startEl, 'dragend', from); + return true; + }, + { from, to, modifierKey } + ); + }) + .catch(() => false); +} + export async function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise { if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) { activeNote.screenshot = await action.saveScreenshot(); diff --git a/src/utils/step-analyzer.ts b/src/utils/step-analyzer.ts index 86b01c0e..be87cac4 100644 --- a/src/utils/step-analyzer.ts +++ b/src/utils/step-analyzer.ts @@ -2,12 +2,13 @@ import type { SessionStep } from '../experience-tracker.ts'; import type { StepData } from '../test-plan.ts'; import { isDynamicId } from './xpath.ts'; -export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const; +export const CODECEPT_TOOLS = ['click', 'hover', 'dragAndDrop', 'pressKey', 'form'] as const; export type CodeceptToolName = (typeof CODECEPT_TOOLS)[number]; const INTERNAL_STEP_PREFIXES = ['grab', 'save']; const CODECEPT_FORM_COMMANDS: readonly string[] = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption']; +const CODECEPT_DRAG_COMMANDS: readonly string[] = ['I.pressKeyDown', 'I.pressKeyUp']; export function isCodeceptToolName(toolName: string): toolName is CodeceptToolName { return CODECEPT_TOOLS.includes(toolName as CodeceptToolName); @@ -17,6 +18,7 @@ export function getCodeceptToolName(commandName: string): CodeceptToolName | nul const toolName = CODECEPT_TOOLS.find((name) => commandName === `I.${name}`); if (toolName) return toolName; if (CODECEPT_FORM_COMMANDS.includes(commandName)) return 'form'; + if (CODECEPT_DRAG_COMMANDS.includes(commandName)) return 'dragAndDrop'; return null; } diff --git a/test-data/drag-board.html b/test-data/drag-board.html new file mode 100644 index 00000000..c23e49b9 --- /dev/null +++ b/test-data/drag-board.html @@ -0,0 +1,67 @@ + + + + + Drag board + + +

Drag board

+ +

Reorderable list

+
    +
  • Item 1
    • +
  • Item 2
    • +
  • Item 3
    • +
+ +

HTML5 drop zone

+
Chip
+
Drop here
+ +

Guarded HTML5 drop zone

+
Guarded chip
+
Guarded drop here
+ + + + diff --git a/tests/integration/drag-and-drop-browser.test.ts b/tests/integration/drag-and-drop-browser.test.ts new file mode 100644 index 00000000..4fd0ed28 --- /dev/null +++ b/tests/integration/drag-and-drop-browser.test.ts @@ -0,0 +1,82 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; +import { type Browser, type Page, chromium } from 'playwright'; +import { syntheticDragDrop } from '../../src/ai/tools.ts'; + +let browser: Browser; + +beforeAll(async () => { + browser = await chromium.launch(); +}); + +afterAll(async () => { + await browser?.close(); +}); + +async function boardPage(): Promise { + const page = await browser.newPage(); + await page.goto(`file://${join(process.cwd(), 'test-data', 'drag-board.html')}`, { waitUntil: 'domcontentloaded' }); + return page; +} + +async function manualDrag(page: Page, source: string, target: string) { + await page.locator(source).hover(); + await page.mouse.down(); + await page.locator(target).hover(); + await page.mouse.up(); +} + +async function listTexts(page: Page): Promise { + return page.locator('#mouse-list li').allTextContents(); +} + +describe('mouse-event drag and drop', () => { + it('reorders the list through the simulated mouse sequence', async () => { + const page = await boardPage(); + + await manualDrag(page, '#mouse-list li:nth-child(2)', '#mouse-list li:nth-child(1)'); + + expect(await listTexts(page)).toEqual(['Item 2', 'Item 1', 'Item 3']); + await page.close(); + }); + + it('duplicates the item when Control is held during the drag', async () => { + const page = await boardPage(); + + await page.locator('#mouse-list li:nth-child(2)').hover(); + await page.keyboard.down('Control'); + await page.mouse.down(); + await page.locator('#mouse-list li:nth-child(1)').hover(); + await page.mouse.up(); + await page.keyboard.up('Control'); + + expect(await listTexts(page)).toEqual(['Item 2 (copy)', 'Item 1', 'Item 2', 'Item 3']); + await page.close(); + }); +}); + +describe('HTML5 drag and drop', () => { + it('responds to the native drag', async () => { + const page = await boardPage(); + + await page.dragAndDrop('#html5-chip', '#drop-zone'); + + expect(await page.locator('#drop-zone #html5-chip').count()).toBe(1); + await page.close(); + }); + + it('needs synthetic drag events when the source cancels the mouse press', async () => { + const page = await boardPage(); + + await manualDrag(page, '#guarded-chip', '#guarded-zone'); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); + + await page.dragAndDrop('#guarded-chip', '#guarded-zone'); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); + + const explorer: any = { withPage: (fn: (page: Page) => Promise) => fn(page) }; + expect(await syntheticDragDrop(explorer, 'I.dragAndDrop("#guarded-chip", "#guarded-zone")')).toBe(true); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(1); + await page.close(); + }); +}); diff --git a/tests/unit/drag-and-drop.test.ts b/tests/unit/drag-and-drop.test.ts new file mode 100644 index 00000000..31554077 --- /dev/null +++ b/tests/unit/drag-and-drop.test.ts @@ -0,0 +1,197 @@ +import { beforeEach, describe, expect, it } from 'bun:test'; +import { createCodeceptJSTools } from '../../src/ai/tools.ts'; +import { ConfigParser } from '../../src/config.ts'; +import { codeceptJSSandbox } from '../../src/utils/web-sandbox.ts'; + +function fakeDeps() { + const holder: { state: any } = { + state: { url: '/board', html: '', ariaSnapshot: '- listitem "First"', id: 'before' }, + }; + const action: any = { + lastError: null, + executedSteps: [] as Array<{ command: string; success: boolean }>, + ran: [] as string[], + saveScreenshot: async () => undefined, + attempt: async (_command: string) => false, + }; + const deps: any = { + explorer: { action: () => action }, + stateManager: { getCurrentState: () => holder.state }, + ai: {}, + }; + return { deps, action, holder }; +} + +function fakeTask(): any { + return { startNote: () => ({ commit: () => {}, screenshot: undefined }) }; +} + +function succeedWithReorder(action: any, holder: { state: any }) { + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + holder.state = { ...holder.state, ariaSnapshot: '- listitem "Second"\n- listitem "First"', id: 'after' }; + return true; + }; +} + +describe('dragAndDrop validation', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + it('rejects empty commands without touching the browser', async () => { + const { deps, action } = fakeDeps(); + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: [], explanation: 'nothing' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('No commands provided'); + expect(action.ran).toEqual([]); + }); + + it('rejects non-dragAndDrop commands as invalid', async () => { + const { deps, action } = fakeDeps(); + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.click("Save")', 'I.dragAndDrop("A", "B")'], explanation: 'drag' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('Invalid commands'); + expect(result.message).toContain('I.click("Save")'); + expect(action.ran).toEqual([]); + }); +}); + +describe('dragAndDrop modifier', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + const block = 'I.pressKeyDown("Control")\nI.dragAndDrop("First", "Second")\nI.pressKeyUp("Control")'; + + it('wraps the drag between pressKeyDown and pressKeyUp and reports the block as code', async () => { + const { deps, action, holder } = fakeDeps(); + succeedWithReorder(action, holder); + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], modifier: 'Control', explanation: 'copy the item' }, {} as any); + + expect(result.success).toBe(true); + expect(result.code).toBe(block); + expect(result.attempts).toEqual([{ command: block, success: true }]); + expect(action.ran).toEqual([block]); + }); + + it('releases the modifier after a failed drag attempt', async () => { + const { deps, action } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + return command === 'I.pressKeyUp("Control")'; + }; + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], modifier: 'Control', explanation: 'copy the item' }, {} as any); + + expect(result.success).toBe(false); + expect(action.ran).toEqual([block, 'I.pressKeyUp("Control")']); + }); + + it('executes the block in order through the CodeceptJS sandbox', () => { + const calls: string[] = []; + const actor: any = new Proxy( + {}, + { + get: + (_t, prop) => + (...args: any[]) => + calls.push(`${String(prop)}:${JSON.stringify(args[0])}`), + } + ); + + codeceptJSSandbox(actor, block); + + expect(calls).toEqual(['pressKeyDown:"Control"', 'dragAndDrop:"First"', 'pressKeyUp:"Control"']); + }); +}); + +describe('dragAndDrop fallbacks', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + it('escalates a silent no-op to the native HTML5 drag and reports it as code', async () => { + const { deps, action, holder } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + if (command.includes('force')) holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; + return true; + }; + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], explanation: 'reorder' }, {} as any); + + expect(result.success).toBe(true); + expect(result.code).toBe('I.dragAndDrop("First", "Second", { force: true })'); + expect(result.attempts).toEqual([ + { command: 'I.dragAndDrop("First", "Second")', success: true }, + { command: 'I.dragAndDrop("First", "Second", { force: true })', success: true }, + ]); + }); + + it('tries the next locator when a drag command fails', async () => { + const { deps, action, holder } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + if (command.includes('.card')) { + action.lastError = Object.assign(new Error('element (.card) was not found by text|CSS|XPath'), { name: 'ElementNotFound' }); + return false; + } + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; + return true; + }; + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop(".card", ".column")', 'I.dragAndDrop("First", "Second")'], explanation: 'reorder' }, {} as any); + + expect(result.success).toBe(true); + expect(result.code).toBe('I.dragAndDrop("First", "Second")'); + expect(result.notExecuted).toBeUndefined(); + }); + + it('falls back to synthetic drag events when both drag mechanisms change nothing', async () => { + const { deps, action, holder } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + return true; + }; + deps.explorer = { + action: () => action, + withPage: async () => { + holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; + return true; + }, + }; + const tools = createCodeceptJSTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("text=Item 2", "text=Item 1")'], explanation: 'reorder' }, {} as any); + + expect(result.success).toBe(true); + expect(result.attempts.map((attempt) => attempt.success)).toEqual([true, true, true]); + expect(result.attempts[2]?.command).toContain('synthetic drag events'); + expect(result.message).toContain('synthetic drag events'); + expect(result.code).toBe('I.dragAndDrop("text=Item 2", "text=Item 1")'); + }); +}); diff --git a/tests/unit/executed-steps.test.ts b/tests/unit/executed-steps.test.ts index c39a237a..dd409514 100644 --- a/tests/unit/executed-steps.test.ts +++ b/tests/unit/executed-steps.test.ts @@ -35,6 +35,18 @@ describe('executed steps', () => { expect(steps[0]?.command).toBe('I.click("Manual", step.opts({"elementIndex":2}))'); }); + it('escapes a locator containing quotes so the recorded command stays valid code', () => { + const steps: ExecutedStep[] = []; + const detachSteps = attachStepLogger(steps); + + const drag = { prefix: '', suffix: '', actor: 'I', name: 'dragAndDrop', args: ['li:has-text("Item 2")', 'li:has-text("Item 1")'], humanizeArgs: () => '"li:has-text("Item 2")"', toCode: () => 'I.dragAndDrop("li:has-text("Item 2")", "li:has-text("Item 1")")' }; + codeceptjs.event.dispatcher.emit(codeceptjs.event.step.passed, drag); + + detachSteps(); + + expect(steps[0]?.command).toBe('I.dragAndDrop("li:has-text(\\"Item 2\\")", "li:has-text(\\"Item 1\\")")'); + }); + it('reports a successful command even when a later one fails', () => { const report = formatExecutedSteps( [ From 01bba2da61d6ef45801062f30a687b64372151a1 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Thu, 8 Oct 2026 19:32:26 +0200 Subject: [PATCH 2/3] UPD --- CHANGELOG.md | 16 +- docs/web-testing/page-interaction.md | 2 +- src/ai/captain/web-mode.ts | 2 +- src/ai/driller.ts | 9 +- src/ai/rerunner.ts | 9 +- src/ai/researcher.ts | 38 +++ src/ai/rules.ts | 24 +- src/ai/tester.ts | 2 +- src/ai/tools.ts | 232 ++++++++---------- src/explorbot.ts | 4 +- .../integration/drag-and-drop-browser.test.ts | 34 +-- tests/unit/drag-and-drop.test.ts | 196 +++++++-------- 12 files changed, 275 insertions(+), 293 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a95f2779..2a8ead58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,18 @@ # Changelog -## 2026-10-06 +## 2026-10-08 ### Changes -- [Tester] New `dragAndDrop()` tool, so scenarios that drag an item onto another item can be tested: reordering lists and trees, moving cards between columns, dropping items into folders or zones. The drag runs as a real mouse simulation first and, when the drop changes nothing, the same drag is retried with the native HTML5 drag and then with synthetic drag events — the last step covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags, and it is released even when the drag fails. Locators go to Playwright as written, so text is written as `text=...` — a bare string is read as CSS, and an object locator loses its text filter. +- [Tester] New `dragAndDrop()` tool, so scenarios that drag something onto something else can be tested: reordering lists and trees, moving cards between columns, dropping items into folders or zones — and dragging slider handles or drawing on canvas, where nothing can be addressed by a locator. The tool takes visual descriptions of what to grab and where to drop it, the vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API. When the drop changes nothing, the same points are retried with HTML5 drag events — covering pages whose drag libraries cancel the mouse press. ```javascript - I.dragAndDrop('text=First chapter', 'text=Second chapter'); // move or reorder + dragAndDrop({ from: 'card labeled Bug 5 in the first column', to: 'column labeled Done' }); // move or reorder ``` - Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records it as plain CodeceptJS — `I.pressKeyDown('Control')`, `I.dragAndDrop(...)`, `I.pressKeyUp('Control')` — so it replays and heals in rerun like any other step. + Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records the drag as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence — so it replays and heals in rerun like any other step. + +## 2026-10-06 + +### Changes - Generated tests no longer break on a locator that contains quotes, such as `li:has-text("Item 2")`: executed commands are recorded with the quotes escaped, so the test file stays valid code and reruns instead of failing to load. - ```javascript - I.dragAndDrop('First chapter', 'Second chapter'); // move or reorder - ``` - Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records it as plain CodeceptJS — `I.pressKeyDown('Control')`, `I.dragAndDrop(...)`, `I.pressKeyUp('Control')` — so it replays and heals in rerun like any other step. ## 2026-10-05 diff --git a/docs/web-testing/page-interaction.md b/docs/web-testing/page-interaction.md index cad1d42b..f18aaa52 100644 --- a/docs/web-testing/page-interaction.md +++ b/docs/web-testing/page-interaction.md @@ -74,7 +74,7 @@ When a locator fails, the agent tries the next strategy, then a visual click. Lo ## Drag and drop -The Tester's `dragAndDrop()` tool drags one element onto another — reordering lists and trees, moving cards between columns, dropping items into folders or zones. It drags with a real mouse simulation first and, when the drop changes nothing, retries the same drag with the native HTML5 drag and then with synthetic drag events on its own — the last step covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags; it is recorded as plain CodeceptJS (`I.pressKeyDown`, `I.dragAndDrop`, `I.pressKeyUp`), so generated tests replay unchanged. +The Tester's `dragAndDrop()` tool takes visual descriptions of what to grab and where to drop it. The vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API — which also covers sliders and canvas drawing, where there is nothing to attach a locator to. When the drop changes nothing, the tool retries the same points with HTML5 drag events on its own; that covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags. The drag is recorded as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence, so generated tests replay unchanged. ## What happens after each action diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index 24972862..fe275f39 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -10,7 +10,7 @@ export function WithWebMode(Base: T) { webModeTools(ctx: ModeContext): Record { const explorer = ctx.explorBot.getExplorer(); const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() }; - const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task); + const codeceptTools = createCodeceptJSTools({ ...toolDeps, researcher: ctx.explorBot.agentResearcher() }, ctx.task); const agentTools = createAgentTools({ ...toolDeps, researcher: ctx.explorBot.agentResearcher(), diff --git a/src/ai/driller.ts b/src/ai/driller.ts index bbc95936..22134d82 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -16,6 +16,7 @@ import { WebElement } from '../utils/web-element.ts'; import type { Agent, AgentDeps } from './agent.ts'; import type { Navigator } from './navigator.ts'; import type { Provider } from './provider.ts'; +import type { Researcher } from './researcher.ts'; import { drillLocatorRule } from './rules.ts'; import { TaskAgent, isInteractive } from './task-agent.ts'; import { createCodeceptJSTools, createLearnExperienceTool } from './tools.ts'; @@ -73,7 +74,11 @@ export class Driller extends TaskAgent implements Agent { MAX_COMPONENT_ITERATIONS = 12; - constructor(deps: AgentDeps, navigator: Navigator) { + constructor( + deps: AgentDeps, + navigator: Navigator, + private researcher?: Researcher + ) { super(deps); this.navigator = navigator; } @@ -293,7 +298,7 @@ export class Driller extends TaskAgent implements Agent { conversation.addUserText(await this.buildComponentPrompt(originalState, component)); let finished = false; - const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component); + const actionTools = this.createVerifiedActionTools(createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, test), component); const learnExperience = createLearnExperienceTool({ getExperienceTracker: () => this.getExperienceTracker(), getState: () => ActionResult.fromState(this.stateManager.getCurrentState() || originalState), diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index d3d3a42c..370f6324 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -24,6 +24,7 @@ import type { Agent, AgentDeps } from './agent.ts'; import { toolExecutionLabel } from './conversation.ts'; import type { Navigator } from './navigator.ts'; import { Provider } from './provider.ts'; +import type { Researcher } from './researcher.ts'; import { actionRule, locatorRule, sectionContextRule } from './rules.ts'; import { TaskAgent } from './task-agent.ts'; import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts'; @@ -35,13 +36,15 @@ export class Rerunner extends TaskAgent implements Agent { emoji = '🔄'; private agentTools: any; + private researcher?: Researcher; private healedSteps: Array<{ original: string; healed: string }> = []; private traceDir = ''; private static pluginsWired = false; - constructor(deps: AgentDeps, agentTools?: any) { + constructor(deps: AgentDeps, agentTools?: any, researcher?: Researcher) { super(deps); this.agentTools = agentTools; + this.researcher = researcher; } protected getNavigator(): Navigator { @@ -312,7 +315,7 @@ export class Rerunner extends TaskAgent implements Agent { }); const healTask = new Task(`Heal: ${failedCode}`); - const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask); + const codeceptTools = createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, healTask); let healed = false; let healedCommand = ''; @@ -441,7 +444,7 @@ export class Rerunner extends TaskAgent implements Agent { - You MUST execute the replacement action — not just diagnose - Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element - - Use dragAndDrop() to drag an element onto another element, with a modifier for copy-drags + - Use dragAndDrop() to drag something onto something else — describe both visually; a modifier makes copy-drags - Use form() for text input, dropdown selection, file uploads - Use pressKey() for special keys or key combinations - Use wait() when page is loading — returns fresh ARIA automatically diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 899e0415..d6bb5ed3 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -567,6 +567,44 @@ export class Researcher extends ResearcherBase implements Agent { return r.text; } + async checkDragPoints(state: WebPageState, fromDescription: string, toDescription: string): Promise { + const screenshotData = this.getScreenshotFromState(state); + if (!screenshotData) return null; + + const { actionResult, image } = screenshotData; + tag('step').log('Locating drag points on screenshot'); + const prompt = dedent` + + You are a precise UI inspector preparing a drag-and-drop operation on a webpage screenshot. + + + + Locate the element to grab: "${fromDescription}". + Locate the drop target: "${toDescription}". + When both are visible, respond with one short sentence naming each and give the point + coordinates: the grab point at the center of what is dragged, the drop point at the center + of where it must land. End the answer with "grab X, Y; drop X, Y" + (numbers must be integers followed by X and Y respectively). + If either element is not visible or cannot be reached, say which one and why — no coordinates then. + Do not describe other elements. + + + + - Keep the answer under three sentences. + - Give both points only when both elements are located. + - Coordinates must follow the pattern "123X, 456Y" with X and Y suffixes. + + + URL: ${actionResult.url || 'Unknown'} + Title: ${actionResult.title || 'Unknown'} + + The screenshot is provided below. + `; + + const r = await this.provider.processImage(prompt, image.toString('base64')); + return r.text; + } + async answerQuestionAboutScreenshot(state: WebPageState, question: string): Promise { const screenshotData = this.getScreenshotFromState(state); if (!screenshotData) return null; diff --git a/src/ai/rules.ts b/src/ai/rules.ts index afcdee19..4f1e6307 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -508,24 +508,20 @@ export const actionRule = dedent` ### I.dragAndDrop Drags an element and drops it onto another element. Run via the dragAndDrop() tool, not form(). - Use for reorderable lists and trees, moving cards between columns, dropping items into folders or zones. - - I.dragAndDrop(, ) - - Unlike I.click, locators here go to Playwright as written: a bare string is read as CSS - and an object locator loses its text filter. Write text as 'text=...' and prefer CSS or XPath. - When an item drags only from a handle inside it, target the handle as the source. - After a result reported several matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument. - For a drag with a modifier held (a copy-drag that duplicates the item), pass the modifier to the dragAndDrop() tool. + The tool takes visual descriptions of what to grab and where to drop it, locates both points + on the screenshot, and drags between them with a real mouse — this also covers sliders and canvas + drawing, where nothing can be addressed by a locator. When items look alike, describe the position + ("the second card in the first column"). When a drag starts from a handle, describe the handle + as the grab point, not the whole card. For a drag with a modifier held (a copy-drag that + duplicates the item), pass the modifier to the dragAndDrop() tool. - I.dragAndDrop('text=First chapter', 'text=Second chapter') - I.dragAndDrop('.card-new', '.column-done') + dragAndDrop(from: "card labeled Bug 5 in the first column", to: "column labeled Done") - The tool drags with a real mouse simulation first and retries with the native HTML5 drag - and then synthetic drag events when the drop changes nothing. When a command fails, the problem - is the locators — change locators, not the mechanism. + When form() must drag directly, I.dragAndDrop(, ) takes Playwright locators + as written: a bare string is read as CSS and an object locator loses its text filter. + Write text as 'text=...' and prefer CSS or XPath. ### I.seeInClipboard / I.seeClipboardEquals / I.clearClipboard diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 2f7bd2c9..e6a3c79c 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -249,7 +249,7 @@ export class Tester extends TaskAgent implements Agent { task.states.push(event.toState); }); - const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task); + const codeceptjsTools = createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, task); let assertionPerformed = false; let extensions = 0; let deadlineReached = false; diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 3b563218..614300dd 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -9,7 +9,6 @@ import { type Task, TestResult } from '../test-plan.js'; import { ariaRefSelector, describeRef, refIsGone } from '../utils/aria-ref.ts'; import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts'; import { isFatalBrowserError } from '../utils/browser-errors.ts'; -import { splitTopLevel } from '../utils/code-extractor.ts'; import { cleanHtmlSnippet } from '../utils/html.ts'; import { createDebug, tag } from '../utils/logger.js'; import { pause } from '../utils/loop.js'; @@ -34,7 +33,7 @@ interface AgentToolDeps extends ToolDeps { export const ASSERTION_TOOLS = ['verify'] as const; -export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDeps, task: Task) { +export function createCodeceptJSTools({ explorer, stateManager, judge, researcher }: ToolDeps & { researcher?: Researcher }, task: Task) { return { click: tool({ description: dedent` @@ -260,139 +259,108 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep dragAndDrop: tool({ description: dedent` - Drag an element and drop it onto another element by trying multiple CodeceptJS commands in order. + Drag something on the page and drop it somewhere else: reorder list or tree items, move cards + between columns, drop into folders or zones, drag slider handles, draw on a canvas. - Use for any drag-and-drop interaction: reordering list or tree items, moving cards between - columns, dropping items into folders or zones. - - I.dragAndDrop(source, target) - drag the element matching the source locator onto the element - matching the target locator. - Unlike I.click, locators here go to Playwright as written: a bare string is read as CSS and - an object locator loses its text filter. Write text as 'text=...' and prefer CSS or XPath. - After a result reported several matches, reuse that locator with step.opts({ elementIndex: N }) - as the last argument. - - The tool drags with a real mouse simulation first and, when the drop changes nothing, retries - the same drag with the native HTML5 drag and then with synthetic drag events — you never choose - between them. When a command fails, the problem is the locators: change locators, not the mechanism. - For a drag with a modifier held (a copy-drag that duplicates the item), pass modifier. - - CRITICAL: All commands MUST drag the SAME element onto the SAME target. - This is a FALLBACK list, NOT a sequence of different drags. + Give visual descriptions of what to grab and where to drop it — the tool locates both points + on the screenshot and drags between them with a real mouse. When items look alike, describe + the position (e.g. "the second card in the first column"). For a copy-drag that duplicates + the item, pass modifier. `, inputSchema: z.object({ - commands: z.array(z.string()).describe(dedent` - FALLBACK commands for ONE drag. All commands must drag the SAME source onto the SAME target. - Each command is a full I.dragAndDrop(source, target) with both locators, ordered by reliability: - 1. I.dragAndDrop('text=First chapter', 'text=Second chapter') - by visible text - 2. I.dragAndDrop(CSS, CSS) - e.g. I.dragAndDrop('.card-new', '.column-done') - 3. I.dragAndDrop(XPath, XPath) - when locators already include context - A bare string is read as CSS, and an object locator loses its text filter — never use them here. - After a result reporting multiple matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument. - `), + from: z.string().describe('What to drag, by visible appearance and position (e.g. "card labeled Bug 5 in the first column", "round handle of the volume slider")'), + to: z.string().describe('Where to drop it, by visible appearance and position (e.g. "column labeled Done", "right end of the volume slider track")'), modifier: z.enum(['Control', 'Shift', 'Alt', 'Meta', 'CommandOrControl']).optional().describe('Modifier key held during the whole drag, e.g. Control for a copy-drag that duplicates the item.'), explanation: z.string().describe('Why you are dragging this element'), }), - execute: async ({ commands: rawCommands, modifier, explanation }) => { + execute: async ({ from, to, modifier, explanation }) => { const activeNote = task.startNote(explanation); - if (rawCommands.length === 0) { + if (!researcher || Stats.visionDisabled) { activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', 'No commands provided'); + return failedToolResult('dragAndDrop', 'Drag points cannot be located in this session. Use form() with I.dragAndDrop(source, target) instead.'); } - const commands = rawCommands.map((cmd) => cmd.trim()); - const invalidCommands = commands.filter((cmd) => !cmd.startsWith('I.dragAndDrop')); + try { + const captured = await explorer.capture({ screenshot: true }); - if (invalidCommands.length > 0) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', `Invalid commands: ${invalidCommands.join(', ')}. Each command must be a full I.dragAndDrop(source, target).`, { - suggestion: 'Use click() to click elements, or form() for other command sequences.', - }); - } + if (!captured.screenshot) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Failed to capture screenshot for locating the drag points'); + } - const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.action(); - const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + const analysis = await researcher.checkDragPoints(captured, from, to); - for (let i = 0; i < commands.length; i++) { - const command = commands[i]; - const block = withModifier(command, modifier); - const success = await action.attempt(block, explanation); + if (!analysis) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Visual analysis failed to process the screenshot'); + } - const attempt: { command: string; success: boolean; error?: string } = { command: block, success }; - if (action.lastError) attempt.error = errorText(action.lastError); - attempts.push(attempt); + const points = dragPoints(analysis); - if (!success) { - await releaseModifier(action, modifier); - continue; + if (!points) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `Drag points not found: ${analysis}`, { + analysis, + suggestion: 'Describe what to drag and where to drop it more precisely, or scroll the target into view and retry.', + }); } - let executedCode = action.executedSteps.map((step) => step.command).join('\n') || block; - let toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, executedCode); + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const dragCode = mouseDragCode(from, to, points); + const block = withModifier(dragCode, modifier); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; - if (!hasObservablePageChange(toolResult) && !command.includes('force')) { - const forceBlock = withModifier(`${command.replace(/\)\s*$/, '')}, { force: true })`, modifier); - const forceSuccess = await action.attempt(forceBlock, explanation); + const mouseSuccess = await action.attempt(block, explanation); + const attempt: { command: string; success: boolean; error?: string } = { command: dragCode, success: mouseSuccess }; + if (!mouseSuccess && action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); - const forceAttempt: { command: string; success: boolean; error?: string } = { command: forceBlock, success: forceSuccess }; - if (!forceSuccess && action.lastError) forceAttempt.error = errorText(action.lastError); - attempts.push(forceAttempt); + if (mouseSuccess) { + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); - if (forceSuccess) { - executedCode = action.executedSteps.map((step) => step.command).join('\n') || forceBlock; - toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, executedCode); - } else { - await releaseModifier(action, modifier); + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block }, action); } } + await releaseModifier(action, modifier); - if (!hasObservablePageChange(toolResult)) { - const dragged = await syntheticDragDrop(explorer, command, modifier); - attempts.push({ command: `${command} (synthetic drag events)`, success: dragged }); - - if (!dragged) { - await releaseModifier(action, modifier); - continue; - } + const dragged = await syntheticDragDrop(explorer, points, modifier); + attempts.push({ command: `${dragCode} (HTML5 drag events)`, success: dragged }); - const replayable = withModifier(command, modifier); - const syntheticResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, replayable); + if (dragged) { + await explorer.capture(); + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); - if (!hasObservablePageChange(syntheticResult)) { - await releaseModifier(action, modifier); - continue; + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block, message: 'The drop landed only after HTML5 drag events were dispatched at the located points; the recorded mouse drag replays through the tool, which runs the same escalation.' }, action); } - - executedCode = replayable; - toolResult = { ...syntheticResult, message: 'The drop landed only after synthetic drag events were dispatched; the recorded command replays through the tool, which runs the same chain again.' }; } - await commitNote(activeNote, TestResult.PASSED, toolResult, action); - const data: Record = { ...toolResult, attempts, code: executedCode }; - const notExecuted = commands.slice(i + 1); - if (notExecuted.length) { - data.notExecuted = notExecuted; - data.suggestion = `SKIPPED: ${notExecuted.join('; ')}`; - } - return successToolResult('dragAndDrop', data, action); + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'dragAndDrop', + 'The drag changed nothing on the page', + { + ...toolResult, + points, + analysis, + attempts, + suggestion: 'Check that what you described is draggable and the target is a valid drop zone. When a drag starts from a handle, describe the handle as the grab point, not the whole card.', + }, + action.lastError, + judge, + explanation + ); + } catch (error) { + throwIfFatalBrowserError(error); + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `dragAndDrop tool failed: ${errorText(error)}`); } - - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]); - await commitNote(activeNote, TestResult.FAILED, toolResult, action); - return failedToolResult( - 'dragAndDrop', - 'All dragAndDrop commands failed', - { - ...toolResult, - attempts, - suggestion: 'A failed drag is a locator problem, not a drag mechanism problem. Check with xpathCheck() that the source and the target each match exactly one element, and that the target is a valid drop zone.', - }, - action.lastError, - judge, - explanation - ); }, }), @@ -1394,40 +1362,26 @@ async function releaseModifier(action: any, modifier?: string): Promise { const SYNTHETIC_MODIFIER_KEYS: Record = { Control: 'ctrlKey', Shift: 'shiftKey', Alt: 'altKey', Meta: 'metaKey', CommandOrControl: 'auto' }; -function unquoteCommandArg(arg: string): string { - if (arg.startsWith('"')) { - try { - return JSON.parse(arg); - } catch { - return arg; - } - } - return arg - .replace(/^['"`]/, '') - .replace(/['"`]$/, '') - .replace(/\\(['"`])/g, '$1'); +export function dragPoints(analysis: string): DragPoints | null { + const match = analysis.match(/grab\s+(\d+)X,\s*(\d+)Y.*?drop\s+(\d+)X,\s*(\d+)Y/is); + if (!match) return null; + return { + from: { x: Number.parseInt(match[1], 10), y: Number.parseInt(match[2], 10) }, + to: { x: Number.parseInt(match[3], 10), y: Number.parseInt(match[4], 10) }, + }; } -function dragLocators(command: string): string[] { - const inner = command.replace(/^I\.dragAndDrop\(/, '').replace(/\)\s*$/, ''); - return splitTopLevel(inner, ',') - .map((part) => part.trim()) - .filter((part) => /^['"`]/.test(part)) - .slice(0, 2) - .map(unquoteCommandArg); +function mouseDragCode(from: string, to: string, points: DragPoints): string { + const { x: fx, y: fy } = points.from; + const { x: tx, y: ty } = points.to; + return `I.usePlaywrightTo(${JSON.stringify(`drag ${from} to ${to}`)}, async ({ page }) => { await page.mouse.move(${fx}, ${fy}); await page.mouse.down(); await page.waitForTimeout(200); await page.mouse.move(${tx}, ${ty}, { steps: 10 }); await page.waitForTimeout(200); await page.mouse.up(); })`; } -export async function syntheticDragDrop(explorer: any, command: string, modifier?: string): Promise { - const [src, dst] = dragLocators(command); - if (!src || !dst) return false; +export async function syntheticDragDrop(explorer: any, points: DragPoints, modifier?: string): Promise { const modifierKey = modifier ? SYNTHETIC_MODIFIER_KEYS[modifier] : undefined; return explorer - .withPage(async (page: any) => { - const [srcBox, dstBox] = await Promise.all([page.locator(src).boundingBox(), page.locator(dst).boundingBox()]); - if (!srcBox || !dstBox) return false; - const from = { x: srcBox.x + srcBox.width / 2, y: srcBox.y + srcBox.height / 2 }; - const to = { x: dstBox.x + dstBox.width / 2, y: dstBox.y + dstBox.height / 2 }; - return page.evaluate( + .withPage(async (page: any) => + page.evaluate( ({ from, to, modifierKey }) => { const startEl = document.elementFromPoint(from.x, from.y); const endEl = document.elementFromPoint(to.x, to.y); @@ -1446,9 +1400,9 @@ export async function syntheticDragDrop(explorer: any, command: string, modifier fire(startEl, 'dragend', from); return true; }, - { from, to, modifierKey } - ); - }) + { from: points.from, to: points.to, modifierKey } + ) + ) .catch(() => false); } @@ -1568,6 +1522,7 @@ export function withdrawVisionTools(tools: Record): void { if (!Stats.visionDisabled) return; Reflect.deleteProperty(tools, 'see'); Reflect.deleteProperty(tools, 'visualClick'); + Reflect.deleteProperty(tools, 'dragAndDrop'); } export function clickFailureSuggestion(attempts: Array<{ error?: string }>): string { @@ -1674,3 +1629,8 @@ interface MatchedElement { label: string; visible?: boolean; } + +export interface DragPoints { + from: { x: number; y: number }; + to: { x: number; y: number }; +} diff --git a/src/explorbot.ts b/src/explorbot.ts index 2c9114ba..a66d5398 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -317,7 +317,7 @@ export class ExplorBot { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false }); - return new Rerunner(deps, tools); + return new Rerunner(deps, tools, researcher); }); if (this.isHistorianEnabled()) this.agents.rerunner.setHistorian(this.agentHistorian()); } @@ -327,7 +327,7 @@ export class ExplorBot { agentDriller(): Driller { return (this.agents.driller ||= this.createAgent((deps) => { const navigator = this.agentNavigator(); - return new Driller(deps, navigator); + return new Driller(deps, navigator, this.agentResearcher()); })); } diff --git a/tests/integration/drag-and-drop-browser.test.ts b/tests/integration/drag-and-drop-browser.test.ts index 4fd0ed28..4b4c1881 100644 --- a/tests/integration/drag-and-drop-browser.test.ts +++ b/tests/integration/drag-and-drop-browser.test.ts @@ -1,7 +1,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; import { join } from 'node:path'; import { type Browser, type Page, chromium } from 'playwright'; -import { syntheticDragDrop } from '../../src/ai/tools.ts'; +import { type DragPoints, syntheticDragDrop } from '../../src/ai/tools.ts'; let browser: Browser; @@ -19,11 +19,21 @@ async function boardPage(): Promise { return page; } -async function manualDrag(page: Page, source: string, target: string) { - await page.locator(source).hover(); +async function centerOf(page: Page, selector: string): Promise<{ x: number; y: number }> { + const box = await page.locator(selector).boundingBox(); + if (!box) throw new Error(`No bounding box for ${selector}`); + return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) }; +} + +async function dragByPoints(page: Page, points: DragPoints, modifier?: string) { + if (modifier) await page.keyboard.down(modifier); + await page.mouse.move(points.from.x, points.from.y); await page.mouse.down(); - await page.locator(target).hover(); + await page.waitForTimeout(200); + await page.mouse.move(points.to.x, points.to.y, { steps: 10 }); + await page.waitForTimeout(200); await page.mouse.up(); + if (modifier) await page.keyboard.up(modifier); } async function listTexts(page: Page): Promise { @@ -31,10 +41,10 @@ async function listTexts(page: Page): Promise { } describe('mouse-event drag and drop', () => { - it('reorders the list through the simulated mouse sequence', async () => { + it('reorders the list through a coordinate mouse drag', async () => { const page = await boardPage(); - await manualDrag(page, '#mouse-list li:nth-child(2)', '#mouse-list li:nth-child(1)'); + await dragByPoints(page, { from: await centerOf(page, '#mouse-list li:nth-child(2)'), to: await centerOf(page, '#mouse-list li:nth-child(1)') }); expect(await listTexts(page)).toEqual(['Item 2', 'Item 1', 'Item 3']); await page.close(); @@ -43,12 +53,7 @@ describe('mouse-event drag and drop', () => { it('duplicates the item when Control is held during the drag', async () => { const page = await boardPage(); - await page.locator('#mouse-list li:nth-child(2)').hover(); - await page.keyboard.down('Control'); - await page.mouse.down(); - await page.locator('#mouse-list li:nth-child(1)').hover(); - await page.mouse.up(); - await page.keyboard.up('Control'); + await dragByPoints(page, { from: await centerOf(page, '#mouse-list li:nth-child(2)'), to: await centerOf(page, '#mouse-list li:nth-child(1)') }, 'Control'); expect(await listTexts(page)).toEqual(['Item 2 (copy)', 'Item 1', 'Item 2', 'Item 3']); await page.close(); @@ -68,14 +73,15 @@ describe('HTML5 drag and drop', () => { it('needs synthetic drag events when the source cancels the mouse press', async () => { const page = await boardPage(); - await manualDrag(page, '#guarded-chip', '#guarded-zone'); + await dragByPoints(page, { from: await centerOf(page, '#guarded-chip'), to: await centerOf(page, '#guarded-zone') }); expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); await page.dragAndDrop('#guarded-chip', '#guarded-zone'); expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); const explorer: any = { withPage: (fn: (page: Page) => Promise) => fn(page) }; - expect(await syntheticDragDrop(explorer, 'I.dragAndDrop("#guarded-chip", "#guarded-zone")')).toBe(true); + const points = { from: await centerOf(page, '#guarded-chip'), to: await centerOf(page, '#guarded-zone') }; + expect(await syntheticDragDrop(explorer, points)).toBe(true); expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(1); await page.close(); }); diff --git a/tests/unit/drag-and-drop.test.ts b/tests/unit/drag-and-drop.test.ts index 31554077..b482d06e 100644 --- a/tests/unit/drag-and-drop.test.ts +++ b/tests/unit/drag-and-drop.test.ts @@ -1,7 +1,8 @@ import { beforeEach, describe, expect, it } from 'bun:test'; -import { createCodeceptJSTools } from '../../src/ai/tools.ts'; +import { createCodeceptJSTools, dragPoints } from '../../src/ai/tools.ts'; import { ConfigParser } from '../../src/config.ts'; -import { codeceptJSSandbox } from '../../src/utils/web-sandbox.ts'; + +const ANALYSIS = 'The first card and the Done column are visible. grab 10X, 20Y; drop 30X, 40Y'; function fakeDeps() { const holder: { state: any } = { @@ -12,14 +13,21 @@ function fakeDeps() { executedSteps: [] as Array<{ command: string; success: boolean }>, ran: [] as string[], saveScreenshot: async () => undefined, - attempt: async (_command: string) => false, + attempt: async (command: string) => { + action.ran.push(command); + return false; + }, + }; + const researcher: any = { + checkDragPoints: async () => ANALYSIS, }; const deps: any = { - explorer: { action: () => action }, + explorer: { action: () => action, capture: async () => ({ screenshot: Buffer.from('png') }) }, stateManager: { getCurrentState: () => holder.state }, ai: {}, + researcher, }; - return { deps, action, holder }; + return { deps, action, holder, researcher }; } function fakeTask(): any { @@ -36,162 +44,128 @@ function succeedWithReorder(action: any, holder: { state: any }) { }; } -describe('dragAndDrop validation', () => { - beforeEach(() => { - ConfigParser.resetForTesting(); - ConfigParser.setupTestConfig(); +describe('dragPoints parsing', () => { + it('reads both points from the vision analysis', () => { + expect(dragPoints('grab 10X, 20Y; drop 30X, 40Y')).toEqual({ + from: { x: 10, y: 20 }, + to: { x: 30, y: 40 }, + }); }); - it('rejects empty commands without touching the browser', async () => { - const { deps, action } = fakeDeps(); - const tools = createCodeceptJSTools(deps, fakeTask()); - - const result = await tools.dragAndDrop.execute({ commands: [], explanation: 'nothing' }, {} as any); - - expect(result.success).toBe(false); - expect(result.message).toContain('No commands provided'); - expect(action.ran).toEqual([]); + it('reads the points from a sentence', () => { + expect(dragPoints('The card is in the first column. grab 5X, 6Y; drop 7X, 8Y')).toEqual({ + from: { x: 5, y: 6 }, + to: { x: 7, y: 8 }, + }); }); - it('rejects non-dragAndDrop commands as invalid', async () => { - const { deps, action } = fakeDeps(); - const tools = createCodeceptJSTools(deps, fakeTask()); - - const result = await tools.dragAndDrop.execute({ commands: ['I.click("Save")', 'I.dragAndDrop("A", "B")'], explanation: 'drag' }, {} as any); - - expect(result.success).toBe(false); - expect(result.message).toContain('Invalid commands'); - expect(result.message).toContain('I.click("Save")'); - expect(action.ran).toEqual([]); + it('returns null when a point is missing', () => { + expect(dragPoints('The drop target was not found')).toBeNull(); + expect(dragPoints('grab 10X, 20Y')).toBeNull(); }); }); -describe('dragAndDrop modifier', () => { +describe('dragAndDrop tool', () => { beforeEach(() => { ConfigParser.resetForTesting(); ConfigParser.setupTestConfig(); }); - const block = 'I.pressKeyDown("Control")\nI.dragAndDrop("First", "Second")\nI.pressKeyUp("Control")'; - - it('wraps the drag between pressKeyDown and pressKeyUp and reports the block as code', async () => { + it('drags between the located points with a real mouse and records replayable code', async () => { const { deps, action, holder } = fakeDeps(); succeedWithReorder(action, holder); const tools = createCodeceptJSTools(deps, fakeTask()); - const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], modifier: 'Control', explanation: 'copy the item' }, {} as any); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); expect(result.success).toBe(true); - expect(result.code).toBe(block); - expect(result.attempts).toEqual([{ command: block, success: true }]); - expect(action.ran).toEqual([block]); + expect(action.ran).toHaveLength(1); + expect(action.ran[0]).toContain('I.usePlaywrightTo'); + expect(action.ran[0]).toContain('page.mouse.move(10, 20)'); + expect(action.ran[0]).toContain('page.mouse.move(30, 40, { steps: 10 })'); + expect(result.code).toBe(action.ran[0]); + expect(result.points).toEqual({ from: { x: 10, y: 20 }, to: { x: 30, y: 40 } }); }); - it('releases the modifier after a failed drag attempt', async () => { - const { deps, action } = fakeDeps(); - action.attempt = async (command: string) => { - action.ran.push(command); - action.lastError = null; - return command === 'I.pressKeyUp("Control")'; - }; + it('wraps the drag between pressKeyDown and pressKeyUp for a modifier', async () => { + const { deps, action, holder } = fakeDeps(); + succeedWithReorder(action, holder); const tools = createCodeceptJSTools(deps, fakeTask()); - const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], modifier: 'Control', explanation: 'copy the item' }, {} as any); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); - expect(result.success).toBe(false); - expect(action.ran).toEqual([block, 'I.pressKeyUp("Control")']); + expect(result.success).toBe(true); + expect(result.code).toBe(action.ran[0]); + expect(result.code.startsWith('I.pressKeyDown("Control")\n')).toBe(true); + expect(result.code.endsWith('I.pressKeyUp("Control")')).toBe(true); }); - it('executes the block in order through the CodeceptJS sandbox', () => { - const calls: string[] = []; - const actor: any = new Proxy( - {}, - { - get: - (_t, prop) => - (...args: any[]) => - calls.push(`${String(prop)}:${JSON.stringify(args[0])}`), - } - ); - - codeceptJSSandbox(actor, block); - - expect(calls).toEqual(['pressKeyDown:"Control"', 'dragAndDrop:"First"', 'pressKeyUp:"Control"']); - }); -}); + it('fails without browser action when vision cannot locate both points', async () => { + const { deps, action } = fakeDeps(); + deps.researcher.checkDragPoints = async () => 'The drop target was not found'; + const tools = createCodeceptJSTools(deps, fakeTask()); -describe('dragAndDrop fallbacks', () => { - beforeEach(() => { - ConfigParser.resetForTesting(); - ConfigParser.setupTestConfig(); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('Drag points not found'); + expect(action.ran).toEqual([]); }); - it('escalates a silent no-op to the native HTML5 drag and reports it as code', async () => { + it('escalates a silent mouse drag to HTML5 drag events', async () => { const { deps, action, holder } = fakeDeps(); action.attempt = async (command: string) => { action.ran.push(command); action.lastError = null; action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); - if (command.includes('force')) holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; return true; }; + let syntheticDone = false; + deps.explorer = { + action: () => action, + capture: async () => { + if (syntheticDone) holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; + return { screenshot: Buffer.from('png') }; + }, + withPage: async () => { + syntheticDone = true; + return true; + }, + }; const tools = createCodeceptJSTools(deps, fakeTask()); - const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("First", "Second")'], explanation: 'reorder' }, {} as any); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); expect(result.success).toBe(true); - expect(result.code).toBe('I.dragAndDrop("First", "Second", { force: true })'); - expect(result.attempts).toEqual([ - { command: 'I.dragAndDrop("First", "Second")', success: true }, - { command: 'I.dragAndDrop("First", "Second", { force: true })', success: true }, - ]); + expect(result.message).toContain('HTML5 drag events'); + expect(result.attempts.map((attempt: any) => attempt.success)).toEqual([true, true]); + expect(result.attempts[1]?.command).toContain('HTML5 drag events'); }); - it('tries the next locator when a drag command fails', async () => { - const { deps, action, holder } = fakeDeps(); + it('releases the modifier and fails when no mechanism changes the page', async () => { + const { deps, action } = fakeDeps(); action.attempt = async (command: string) => { action.ran.push(command); - if (command.includes('.card')) { - action.lastError = Object.assign(new Error('element (.card) was not found by text|CSS|XPath'), { name: 'ElementNotFound' }); - return false; - } - action.lastError = null; - action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); - holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; - return true; + return command.startsWith('I.pressKeyUp'); }; + deps.explorer.withPage = async () => false; const tools = createCodeceptJSTools(deps, fakeTask()); - const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop(".card", ".column")', 'I.dragAndDrop("First", "Second")'], explanation: 'reorder' }, {} as any); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); - expect(result.success).toBe(true); - expect(result.code).toBe('I.dragAndDrop("First", "Second")'); - expect(result.notExecuted).toBeUndefined(); + expect(result.success).toBe(false); + expect(result.message).toContain('changed nothing'); + expect(action.ran[action.ran.length - 1]).toBe('I.pressKeyUp("Control")'); }); - it('falls back to synthetic drag events when both drag mechanisms change nothing', async () => { - const { deps, action, holder } = fakeDeps(); - action.attempt = async (command: string) => { - action.ran.push(command); - action.lastError = null; - action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); - return true; - }; - deps.explorer = { - action: () => action, - withPage: async () => { - holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; - return true; - }, - }; - const tools = createCodeceptJSTools(deps, fakeTask()); + it('reports drag as unavailable when there is no researcher', async () => { + const { deps, action } = fakeDeps(); + const tools = createCodeceptJSTools({ ...deps, researcher: undefined }, fakeTask()); - const result = await tools.dragAndDrop.execute({ commands: ['I.dragAndDrop("text=Item 2", "text=Item 1")'], explanation: 'reorder' }, {} as any); + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); - expect(result.success).toBe(true); - expect(result.attempts.map((attempt) => attempt.success)).toEqual([true, true, true]); - expect(result.attempts[2]?.command).toContain('synthetic drag events'); - expect(result.message).toContain('synthetic drag events'); - expect(result.code).toBe('I.dragAndDrop("text=Item 2", "text=Item 1")'); + expect(result.success).toBe(false); + expect(result.message).toContain('cannot be located'); + expect(action.ran).toEqual([]); }); }); From 5f1c6219f9c46614ba53cdae7c0041f515a50a75 Mon Sep 17 00:00:00 2001 From: Denys Kuchma Date: Mon, 9 Nov 2026 18:22:59 +0100 Subject: [PATCH 3/3] Rework dragAndDrop into opt-in explicit tools with planner and pilot gates --- CHANGELOG.md | 5 + docs/web-testing/page-interaction.md | 9 +- src/ai/captain/web-mode.ts | 2 +- src/ai/driller.ts | 11 +- src/ai/explicit-tools.ts | 284 ++++++++++++++++++ src/ai/pilot.ts | 9 +- src/ai/planner.ts | 14 + src/ai/rerunner.ts | 12 +- src/ai/rules.ts | 43 ++- src/ai/tester.ts | 43 ++- src/ai/tools.ts | 176 +---------- src/explorbot.ts | 4 +- src/test-plan.ts | 1 + src/utils/test-plan-markdown.ts | 12 +- .../integration/drag-and-drop-browser.test.ts | 2 +- tests/integration/planner.test.ts | 17 ++ tests/unit/drag-and-drop.test.ts | 20 +- tests/unit/explicit-tools.test.ts | 200 ++++++++++++ 18 files changed, 639 insertions(+), 225 deletions(-) create mode 100644 src/ai/explicit-tools.ts create mode 100644 tests/unit/explicit-tools.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e912c35..b1678207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -238,6 +238,11 @@ - An `explore` run ends with an "Explore more" section suggesting `explorbot explore /` to cover the whole site, plus a link to the docs. - Init makes it explicit how to connect the AI: the `.env` template and the missing-key warning point to the provider's key page (e.g. `https://openrouter.ai/keys`) and show the exact line to paste — `OPENROUTER_API_KEY=`; the setup wizard shows where to create the key too. +## 2026-11-09 + +### Changes +- [Tester] Non-standard interaction tools — `dragAndDrop()` and `rightClick()` — are now explicit tools: off by default, so they never grow the Tester's context for ordinary scenarios. The Planner decides per scenario (via the decision model) whether a drag or a context menu is needed and arms the tools for exactly those scenarios; the choice is stored in the plan file (`tools:` line in the `` comment) and survives re-runs. Mid-run, the Pilot can also arm the tools by including `ENABLE_TOOLS: ` in its instruction when a scenario turns out to need an interaction `click()`/`form()` cannot express. + ## 2026-10-08 ### Changes diff --git a/docs/web-testing/page-interaction.md b/docs/web-testing/page-interaction.md index f18aaa52..67dc88b2 100644 --- a/docs/web-testing/page-interaction.md +++ b/docs/web-testing/page-interaction.md @@ -72,9 +72,14 @@ The agent starts with short semantic locators from ARIA or visible text. When a When a locator fails, the agent tries the next strategy, then a visual click. Locators that worked are saved to experience and preferred on the next run. -## Drag and drop +## Explicit tools: drag and drop, right-click -The Tester's `dragAndDrop()` tool takes visual descriptions of what to grab and where to drop it. The vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API — which also covers sliders and canvas drawing, where there is nothing to attach a locator to. When the drop changes nothing, the tool retries the same points with HTML5 drag events on its own; that covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags. The drag is recorded as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence, so generated tests replay unchanged. +Some interactions — dragging an element, opening a context menu with a right-click — need dedicated tools (`dragAndDrop`, `rightClick` in `src/ai/explicit-tools.ts`). They are **off by default**, so they never grow the Tester's context for ordinary scenarios. Two gates turn them on: + +- **Planner gate** — during planning, each scenario is checked with the decision model (`provider.oracle()`, with the agentic model as fallback). A scenario that needs a drag or a context menu gets the tools armed; the choice is stored in the plan (`tools:` in the `` comment) and survives re-runs. +- **Pilot gate** — mid-run, the Pilot can request the tools for the Tester by including `ENABLE_TOOLS: ` in its instruction, when the scenario turns out to need an interaction `click()`/`form()` cannot express. + +The `dragAndDrop()` tool takes visual descriptions of what to grab and where to drop it. The vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API — which also covers sliders and canvas drawing, where there is nothing to attach a locator to. When the drop changes nothing, the tool retries the same points with HTML5 drag events on its own; that covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags. The drag is recorded as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence, so generated tests replay unchanged. ## What happens after each action diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index bfb27978..c667f205 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -146,7 +146,7 @@ export function WithWebMode(Base: T) { webModePrompt(): string { return dedent` - - Page actions: click, dragAndDrop, pressKey, form (CodeceptJS tools) + - Page actions: click, pressKey, form (CodeceptJS tools) - Navigation: navigate() — AI-powered navigation to URLs or page descriptions - Browser diagnostics: browser() — inspect status, evaluate JS, close tabs, screenshot, reload, recover closed/crashed pages, restart browser, open a fresh tab - Visual analysis: see() — screenshot-based page verification diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 4ea9b41a..533a3eb3 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -15,7 +15,6 @@ import { eidxInContainer } from '../utils/web-eidx.ts'; import { WebElement } from '../utils/web-element.ts'; import type { Agent, AgentDeps } from './agent.ts'; import type { Navigator } from './navigator.ts'; -import type { Researcher } from './researcher.ts'; import { drillLocatorRule } from './rules.ts'; import { TaskAgent, isInteractive } from './task-agent.ts'; import { createCodeceptJSTools, createLearnExperienceTool } from './tools.ts'; @@ -63,7 +62,7 @@ interface DrillOptions { } export class Driller extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'dragAndDrop', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = 'D'; private navigator: Navigator; private currentPlan?: Plan; @@ -73,11 +72,7 @@ export class Driller extends TaskAgent implements Agent { MAX_COMPONENT_ITERATIONS = 12; - constructor( - deps: AgentDeps, - navigator: Navigator, - private researcher?: Researcher - ) { + constructor(deps: AgentDeps, navigator: Navigator) { super(deps); this.navigator = navigator; } @@ -297,7 +292,7 @@ export class Driller extends TaskAgent implements Agent { conversation.addUserText(await this.buildComponentPrompt(originalState, component)); let finished = false; - const actionTools = this.createVerifiedActionTools(createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, test), component); + const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.toolDeps, test), component); const learnExperience = createLearnExperienceTool({ getExperienceTracker: () => this.getExperienceTracker(), getState: () => ActionResult.fromState(this.stateManager.getCurrentState() || originalState), diff --git a/src/ai/explicit-tools.ts b/src/ai/explicit-tools.ts new file mode 100644 index 00000000..d2351f0c --- /dev/null +++ b/src/ai/explicit-tools.ts @@ -0,0 +1,284 @@ +import { tool } from 'ai'; +import dedent from 'dedent'; +import { z } from 'zod'; +import { ActionResult } from '../action-result.ts'; +import { Stats } from '../stats.ts'; +import { type Task, TestResult } from '../test-plan.js'; +import type { ToolDeps } from './agent.ts'; +import type { Researcher } from './researcher.ts'; +import { clickFailureSuggestion, commitNote, errorText, failedToolResult, hasObservablePageChange, successToolResult, throwIfFatalBrowserError } from './tools.ts'; + +export const EXPLICIT_TOOL_NAMES = ['dragAndDrop', 'rightClick'] as const; + +interface ExplicitToolDeps extends ToolDeps { + researcher?: Researcher; +} + +export function createExplicitTools({ explorer, stateManager, judge, researcher }: ExplicitToolDeps, task: Task): Record { + return { + dragAndDrop: tool({ + description: dedent` + Drag something on the page and drop it somewhere else: reorder list or tree items, move cards + between columns, drop into folders or zones, drag slider handles, draw on a canvas. + + Give visual descriptions of what to grab and where to drop it — the tool locates both points + on the screenshot and drags between them with a real mouse. When items look alike, describe + the position (e.g. "the second card in the first column"). For a copy-drag that duplicates + the item, pass modifier. + `, + inputSchema: z.object({ + from: z.string().describe('What to drag, by visible appearance and position (e.g. "card labeled Bug 5 in the first column", "round handle of the volume slider")'), + to: z.string().describe('Where to drop it, by visible appearance and position (e.g. "column labeled Done", "right end of the volume slider track")'), + modifier: z.enum(['Control', 'Shift', 'Alt', 'Meta', 'CommandOrControl']).optional().describe('Modifier key held during the whole drag, e.g. Control for a copy-drag that duplicates the item.'), + explanation: z.string().describe('Why you are dragging this element'), + }), + execute: async ({ from, to, modifier, explanation }) => { + const activeNote = task.startNote(explanation); + + if (!researcher || Stats.visionDisabled) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Drag points cannot be located in this session. Use form() with I.dragAndDrop(source, target) instead.'); + } + + try { + const captured = await explorer.capture({ screenshot: true }); + + if (!captured.screenshot) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Failed to capture screenshot for locating the drag points'); + } + + const analysis = await researcher.checkDragPoints(captured, from, to); + + if (!analysis) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Visual analysis failed to process the screenshot'); + } + + const points = dragPoints(analysis); + + if (!points) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `Drag points not found: ${analysis}`, { + analysis, + suggestion: 'Describe what to drag and where to drop it more precisely, or scroll the target into view and retry.', + }); + } + + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const dragCode = mouseDragCode(from, to, points); + const block = withModifier(dragCode, modifier); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + + const mouseSuccess = await action.attempt(block, explanation); + const attempt: { command: string; success: boolean; error?: string } = { command: dragCode, success: mouseSuccess }; + if (!mouseSuccess && action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); + + if (mouseSuccess) { + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block }, action); + } + } + await releaseModifier(action, modifier); + + const dragged = await syntheticDragDrop(explorer, points, modifier); + attempts.push({ command: `${dragCode} (HTML5 drag events)`, success: dragged }); + + if (dragged) { + await explorer.capture(); + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block, message: 'The drop landed only after HTML5 drag events were dispatched at the located points; the recorded mouse drag replays through the tool, which runs the same escalation.' }, action); + } + } + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'dragAndDrop', + 'The drag changed nothing on the page', + { + ...toolResult, + points, + analysis, + attempts, + suggestion: 'Check that what you described is draggable and the target is a valid drop zone. When a drag starts from a handle, describe the handle as the grab point, not the whole card.', + }, + action.lastError, + judge, + explanation + ); + } catch (error) { + throwIfFatalBrowserError(error); + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `dragAndDrop tool failed: ${errorText(error)}`); + } + }, + }), + + rightClick: tool({ + description: dedent` + Right-click an element to open its context menu, then act on the menu items it reveals. + + This tool ONLY accepts I.rightClick() commands. After the menu opens, click its items + with click() — the menu is not part of this tool. + `, + inputSchema: z.object({ + commands: z.array(z.string()).describe(dedent` + FALLBACK LOCATORS for ONE element. All commands must right-click the SAME element. + Order by reliability: + 1. I.rightClick(text, container) + 2. I.rightClick(ARIA, container) + 3. I.rightClick(CSS, container) + 4. I.rightClick(CSS) or I.rightClick(XPath) + `), + explanation: z.string().describe('Why you are right-clicking this element'), + }), + execute: async ({ commands: rawCommands, explanation }) => { + const activeNote = task.startNote(explanation); + + if (rawCommands.length === 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', 'No commands provided'); + } + + const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.rightClick')); + + if (invalidCommands.length > 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', `Invalid commands: ${invalidCommands.join(', ')}. rightClick tool only accepts I.rightClick() commands.`, { + suggestion: 'Use click() for normal clicks, or form() for multiple actions.', + }); + } + + const commands = rawCommands.map((cmd) => { + const trimmed = cmd.trim(); + if (trimmed.startsWith('I.rightClick')) return trimmed; + return `I.rightClick(${JSON.stringify(trimmed)})`; + }); + + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + + for (const command of commands) { + const success = await action.attempt(command, explanation); + const attempt: { command: string; success: boolean; error?: string } = { command, success }; + if (action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); + + if (!success) continue; + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command); + + if (!hasObservablePageChange(toolResult)) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', 'Right-click executed, but no context menu appeared.', { + ...toolResult, + attempts, + code: command, + suggestion: 'Treat the element as not right-clickable. It may not attach a context menu, or the menu opens on hover instead.', + }); + } + + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('rightClick', { ...toolResult, attempts, code: command }, action); + } + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'rightClick', + 'All right-click commands failed', + { + ...toolResult, + attempts, + suggestion: clickFailureSuggestion(attempts), + }, + action.lastError, + judge, + explanation + ); + }, + }), + }; +} + +export function normalizeExplicitTools(names: string[]): string[] { + const canonical = names.map((name) => EXPLICIT_TOOL_NAMES.find((known) => known.toLowerCase() === name.trim().toLowerCase())).filter(Boolean) as string[]; + return [...new Set(canonical)]; +} + +export function parseExplicitToolsRequest(text: string): string[] { + const match = text.match(/^ENABLE_TOOLS:\s*([A-Za-z, ]+)$/m); + if (!match) return []; + return normalizeExplicitTools(match[1].split(',')); +} + +function withModifier(command: string, modifier?: string): string { + if (!modifier) return command; + return `I.pressKeyDown(${JSON.stringify(modifier)})\n${command}\nI.pressKeyUp(${JSON.stringify(modifier)})`; +} + +async function releaseModifier(action: any, modifier?: string): Promise { + if (!modifier) return; + await action.attempt(`I.pressKeyUp(${JSON.stringify(modifier)})`, `Release the ${modifier} key left held by a failed drag`); +} + +const SYNTHETIC_MODIFIER_KEYS: Record = { Control: 'ctrlKey', Shift: 'shiftKey', Alt: 'altKey', Meta: 'metaKey', CommandOrControl: 'auto' }; + +export function dragPoints(analysis: string): DragPoints | null { + const match = analysis.match(/grab\s+(\d+)X,\s*(\d+)Y.*?drop\s+(\d+)X,\s*(\d+)Y/is); + if (!match) return null; + return { + from: { x: Number.parseInt(match[1], 10), y: Number.parseInt(match[2], 10) }, + to: { x: Number.parseInt(match[3], 10), y: Number.parseInt(match[4], 10) }, + }; +} + +function mouseDragCode(from: string, to: string, points: DragPoints): string { + const { x: fx, y: fy } = points.from; + const { x: tx, y: ty } = points.to; + return `I.usePlaywrightTo(${JSON.stringify(`drag ${from} to ${to}`)}, async ({ page }) => { await page.mouse.move(${fx}, ${fy}); await page.mouse.down(); await page.waitForTimeout(200); await page.mouse.move(${tx}, ${ty}, { steps: 10 }); await page.waitForTimeout(200); await page.mouse.up(); })`; +} + +export async function syntheticDragDrop(explorer: any, points: DragPoints, modifier?: string): Promise { + const modifierKey = modifier ? SYNTHETIC_MODIFIER_KEYS[modifier] : undefined; + return explorer + .withPage(async (page: any) => + page.evaluate( + ({ from, to, modifierKey }: { from: { x: number; y: number }; to: { x: number; y: number }; modifierKey?: string }) => { + const startEl = document.elementFromPoint(from.x, from.y); + const endEl = document.elementFromPoint(to.x, to.y); + if (!startEl || !endEl) return false; + const key = modifierKey === 'auto' ? (navigator.platform.includes('Mac') ? 'metaKey' : 'ctrlKey') : modifierKey; + const dataTransfer = new DataTransfer(); + const fire = (el: Element, type: string, at: { x: number; y: number }) => { + const init: any = { bubbles: true, cancelable: true, dataTransfer, clientX: at.x, clientY: at.y }; + if (key) init[key] = true; + el.dispatchEvent(new DragEvent(type, init)); + }; + fire(startEl, 'dragstart', from); + fire(endEl, 'dragenter', to); + fire(endEl, 'dragover', to); + fire(endEl, 'drop', to); + fire(startEl, 'dragend', from); + return true; + }, + { from: points.from, to: points.to, modifierKey } + ) + ) + .catch(() => false); +} + +export interface DragPoints { + from: { x: number; y: number }; + to: { x: number; y: number }; +} diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index b26e44e4..e4a9b0bf 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1264,8 +1264,13 @@ export class Pilot implements Agent { To request more context, mention ATTACH_ARIA, ATTACH_SUMMARY, or ATTACH_UI_MAP — only when recent actions show failures. - Tester tools: click, hover, dragAndDrop, pressKey, form, see, verify, interact, context, research, xpathCheck, - visualClick, back, getVisitedStates, reset, stop, finish, record. + dragAndDrop and rightClick are extra tools: Tester has them only when the plan enables them or after you request them. + Request by including a line "ENABLE_TOOLS: " in your instruction — when the scenario needs + dragging an element or opening a context menu and click()/form() cannot achieve it (for example, repeated failed + attempts to move an item). + + Tester tools: click, hover, pressKey, form, see, verify, interact, context, research, xpathCheck, + visualClick, back, getVisitedStates, reset, stop, finish, record; extra when enabled: dragAndDrop, rightClick. Use tool names exactly as listed. Do not invent combined names or aliases. Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.refreshPage() through form. diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 30ccae23..f8a2f687 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -17,6 +17,7 @@ import { RulesLoader } from '../utils/rules-loader.ts'; import { findPlanMarkdownProblems, parsePlansFromMarkdownText, planToCompactAiContext, readPlanMarkdownComment } from '../utils/test-plan-markdown.ts'; import type { Agent, AgentDeps } from './agent.js'; import { Conversation } from './conversation.ts'; +import { EXPLICIT_TOOL_NAMES } from './explicit-tools.ts'; import type { Fisherman } from './fisherman.ts'; import { WithSessionDedup } from './planner/session-dedup.ts'; import { getActiveStyle } from './planner/styles.ts'; @@ -244,6 +245,10 @@ export class Planner extends PlannerBase implements Agent { const tests = result.tests; debugLog('Created tests:', tests); + for (const test of tests) { + if (await this.needsExplicitTools(test)) test.explicitTools = [...EXPLICIT_TOOL_NAMES]; + } + if (!this.currentPlan) { const cached = state.url ? getRegisteredPlan(state.url) : null; const planName = feature || cached?.plan.title || result.planName || state.url; @@ -282,6 +287,15 @@ export class Planner extends PlannerBase implements Agent { return this.lastSuite; } + private async needsExplicitTools(test: Test): Promise { + const needs = await this.provider.oracle( + 'Performing the steps of this test scenario or verifying its expected outcomes requires a non-standard interaction: dragging an element to a new position, or opening a context menu with a right-click.', + { scenario: test.scenario, steps: test.plannedSteps, expectedOutcomes: test.expected }, + { agentName: 'planner', telemetry: { functionId: 'planner.explicitTools' } } + ); + return needs === true; + } + private addNewTests(tests: Test[], defaultStartUrl: string): Test[] { if (!this.currentPlan) return []; diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index a16946e4..5bfe6a04 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -23,7 +23,6 @@ import { ensureCodeceptResolvable } from '../utils/test-files.ts'; import type { Agent, AgentDeps } from './agent.ts'; import { toolExecutionLabel } from './conversation.ts'; import type { Navigator } from './navigator.ts'; -import type { Researcher } from './researcher.ts'; import { actionRule, locatorRule, sectionContextRule } from './rules.ts'; import { TaskAgent } from './task-agent.ts'; import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts'; @@ -31,19 +30,17 @@ import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts'; const debugLog = createDebug('explorbot:rerunner'); export class Rerunner extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'dragAndDrop', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; emoji = '🔄'; private agentTools: any; - private researcher?: Researcher; private healedSteps: Array<{ original: string; healed: string }> = []; private traceDir = ''; private static pluginsWired = false; - constructor(deps: AgentDeps, agentTools?: any, researcher?: Researcher) { + constructor(deps: AgentDeps, agentTools?: any) { super(deps); this.agentTools = agentTools; - this.researcher = researcher; } protected getNavigator(): Navigator { @@ -314,7 +311,7 @@ export class Rerunner extends TaskAgent implements Agent { }); const healTask = new Task(`Heal: ${failedCode}`); - const codeceptTools = createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, healTask); + const codeceptTools = createCodeceptJSTools(this.toolDeps, healTask); let healed = false; let healedCommand = ''; @@ -443,8 +440,7 @@ export class Rerunner extends TaskAgent implements Agent { - You MUST execute the replacement action — not just diagnose - Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element - - Use dragAndDrop() to drag something onto something else — describe both visually; a modifier makes copy-drags - - Use form() for text input, dropdown selection, file uploads + - Use form() for text input, dropdown selection, file uploads; drag steps are recorded as plain I.usePlaywrightTo mouse code - Use pressKey() for special keys or key combinations - Use wait() when page is loading — returns fresh ARIA automatically - Use research() to understand page structure, sections, and available UI elements diff --git a/src/ai/rules.ts b/src/ai/rules.ts index a18a5509..7e412c17 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -506,18 +506,6 @@ export const actionRule = dedent` ### I.dragAndDrop - Drags an element and drops it onto another element. Run via the dragAndDrop() tool, not form(). - The tool takes visual descriptions of what to grab and where to drop it, locates both points - on the screenshot, and drags between them with a real mouse — this also covers sliders and canvas - drawing, where nothing can be addressed by a locator. When items look alike, describe the position - ("the second card in the first column"). When a drag starts from a handle, describe the handle - as the grab point, not the whole card. For a drag with a modifier held (a copy-drag that - duplicates the item), pass the modifier to the dragAndDrop() tool. - - - dragAndDrop(from: "card labeled Bug 5 in the first column", to: "column labeled Done") - - When form() must drag directly, I.dragAndDrop(, ) takes Playwright locators as written: a bare string is read as CSS and an object locator loses its text filter. Write text as 'text=...' and prefer CSS or XPath. @@ -541,3 +529,34 @@ export const actionRule = dedent` `; + +export const explicitToolsRule = dedent` + + These tools are present only when the plan enables them or the Pilot requests them mid-run. + They cover interactions click() and form() cannot express. + + ### dragAndDrop() tool + + Drags an element and drops it onto another element. Run via the dragAndDrop() tool, not form(). + The tool takes visual descriptions of what to grab and where to drop it, locates both points + on the screenshot, and drags between them with a real mouse — this also covers sliders and canvas + drawing, where nothing can be addressed by a locator. When items look alike, describe the position + ("the second card in the first column"). When a drag starts from a handle, describe the handle + as the grab point, not the whole card. For a drag with a modifier held (a copy-drag that + duplicates the item), pass the modifier to the dragAndDrop() tool. + + + dragAndDrop(from: "card labeled Bug 5 in the first column", to: "column labeled Done") + + + ### rightClick() tool + + Opens a context menu on an element with a right-click. Run via the rightClick() tool, not form(). + All commands in one call must target the SAME element, ordered by locator reliability like click(). + After the menu opens, act on its items with click() — the menu items are not part of this tool. + + + rightClick(commands: ['I.rightClick("Bug 5", ".board")']) + + + `; diff --git a/src/ai/tester.ts b/src/ai/tester.ts index af6cca04..4923d7da 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -18,11 +18,12 @@ import { compactErrorMessage } from '../utils/strings.ts'; import { isSamePageIgnoringQuery } from '../utils/url-matcher.ts'; import type { Agent, AgentDeps } from './agent.ts'; import type { Conversation } from './conversation.ts'; +import { createExplicitTools, normalizeExplicitTools, parseExplicitToolsRequest } from './explicit-tools.ts'; import { pageEvidence } from './judge.ts'; import { Navigator } from './navigator.ts'; import type { Pilot } from './pilot.ts'; import { Researcher } from './researcher.ts'; -import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts'; +import { actionRule, capabilityGroundingRule, dataProtectionRules, explicitToolsRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts'; import { TaskAgent } from './task-agent.ts'; import { createCodeceptJSTools, createIframeTools, withdrawVisionTools } from './tools.ts'; @@ -40,7 +41,7 @@ const SAMPLE_FILES: Record = { }; export class Tester extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'hover', 'dragAndDrop', 'pressKey', 'form']; + protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form']; protected readonly DELEGATED_ACTION_TOOLS = ['interact', 'visualClick']; protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe']; emoji = '🧪'; @@ -64,6 +65,7 @@ export class Tester extends TaskAgent implements Agent { private seenUiMapUrls = new Set(); private lastAnalyzedStateHash: string | null = null; private stalledIterations = 0; + private enabledExplicitTools = new Set(); private previousRegionPresent: boolean | null = null; private regionTransitioned = false; private readonly MAX_STALLED_ITERATIONS = 3; @@ -120,6 +122,7 @@ export class Tester extends TaskAgent implements Agent { this.seenUiMapUrls.clear(); this.lastAnalyzedStateHash = null; this.stalledIterations = 0; + this.enabledExplicitTools = new Set(normalizeExplicitTools(task.explicitTools)); this.pendingReview = ''; this.previousRegionPresent = null; this.regionTransitioned = false; @@ -244,7 +247,10 @@ export class Tester extends TaskAgent implements Agent { const pageContext = await this.reinjectContextIfNeeded(1, testerState); if (pageContext) conversation.addUserText(pageContext); - if (plan) conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`); + if (plan) { + const enabledRules = this.applyExplicitTools(plan); + conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.${enabledRules}`); + } const offStateChange = this.stateManager.onStateChange((event: StateTransition) => { if (task.hasFinished) return; @@ -253,7 +259,8 @@ export class Tester extends TaskAgent implements Agent { task.states.push(event.toState); }); - const codeceptjsTools = createCodeceptJSTools({ ...this.toolDeps, researcher: this.researcher }, task); + const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task); + const explicitTools = createExplicitTools({ ...this.toolDeps, researcher: this.researcher }, task); let assertionPerformed = false; let extensions = 0; let deadlineReached = false; @@ -282,6 +289,10 @@ export class Tester extends TaskAgent implements Agent { Object.assign(tools, createIframeTools(this.toolDeps)); } withdrawVisionTools(tools); + for (const name of this.enabledExplicitTools) { + if (Stats.visionDisabled && name === 'dragAndDrop') continue; + if (explicitTools[name]) tools[name] = explicitTools[name]; + } debugLog(`Test ${task.scenario} iteration ${iteration}`); @@ -320,7 +331,7 @@ export class Tester extends TaskAgent implements Agent { if (isNewPage && this.pilot) { const guidance = await this.pilot.reviewNewPage(task, currentState, conversation); - if (guidance) nextStep += `\n\n${guidance}`; + if (guidance) nextStep += `\n\n${guidance}${this.applyExplicitTools(guidance)}`; } else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) { let guidance: string; if (this.isStruggling) { @@ -328,7 +339,7 @@ export class Tester extends TaskAgent implements Agent { } else { guidance = await this.pilot.periodicAnalyzeProgress(task, currentState, conversation); } - if (guidance) nextStep += `\n\n${guidance}`; + if (guidance) nextStep += `\n\n${guidance}${this.applyExplicitTools(guidance)}`; this.lastAnalyzedStateHash = currentState.hash; } conversation.addUserText(nextStep); @@ -350,7 +361,7 @@ export class Tester extends TaskAgent implements Agent { const allToolNames = result?.toolExecutions?.map((execution: any) => execution.toolName) || []; const successfulToolNames = result?.toolExecutions?.filter((execution: any) => execution.wasSuccessful)?.map((execution: any) => execution.toolName) || []; - const actionPerformed = !!allToolNames.find((toolName: string) => this.ACTION_TOOLS.includes(toolName)); + const actionPerformed = !!allToolNames.find((toolName: string) => this.currentActionTools.includes(toolName)); assertionPerformed = !!successfulToolNames.find((toolName: string) => this.ASSERTION_TOOLS.includes(toolName)); const wasSuccessful = result?.toolExecutions?.every((execution: any) => execution.wasSuccessful); @@ -492,10 +503,21 @@ export class Tester extends TaskAgent implements Agent { return this.stalledIterations >= this.MAX_STALLED_ITERATIONS - 1; } + private get currentActionTools(): string[] { + return [...this.ACTION_TOOLS, ...this.enabledExplicitTools]; + } + + private applyExplicitTools(guidance: string): string { + const requested = parseExplicitToolsRequest(guidance).filter((name) => !this.enabledExplicitTools.has(name)); + if (requested.length === 0) return ''; + for (const name of requested) this.enabledExplicitTools.add(name); + return `\n\n${explicitToolsRule}`; + } + private shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean { const currentState = this.getCurrentState(); const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash; - const actionTools = [...this.ACTION_TOOLS, ...this.DELEGATED_ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS]; + const actionTools = [...this.currentActionTools, ...this.DELEGATED_ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS]; const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName)); const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName)); @@ -514,7 +536,7 @@ export class Tester extends TaskAgent implements Agent { - Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page. + Use tools ${this.currentActionTools.join(', ')} to interact with the page. Fall back to interact() when those fail, when the step needs a sequence of actions, or when your context is not enough to locate the element. Use tool names exactly as listed in this prompt. Do not invent combined tool names or aliases. Match each tool input schema exactly. Do not invent parameter names or pass extra fields. @@ -743,6 +765,7 @@ export class Tester extends TaskAgent implements Agent { `; } + const enabledExplicitRule = this.enabledExplicitTools.size > 0 ? explicitToolsRule : ''; return dedent` @@ -837,6 +860,8 @@ export class Tester extends TaskAgent implements Agent { ${actionRule} + ${enabledExplicitRule} + ${sectionContextRule} ${formRequirementsRule} diff --git a/src/ai/tools.ts b/src/ai/tools.ts index f07ddeec..ced3620a 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -33,7 +33,7 @@ interface AgentToolDeps extends ToolDeps { export const ASSERTION_TOOLS = ['verify'] as const; -export function createCodeceptJSTools({ explorer, stateManager, judge, researcher }: ToolDeps & { researcher?: Researcher }, task: Task) { +export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDeps, task: Task) { return { click: tool({ description: dedent` @@ -257,113 +257,6 @@ export function createCodeceptJSTools({ explorer, stateManager, judge, researche }, }), - dragAndDrop: tool({ - description: dedent` - Drag something on the page and drop it somewhere else: reorder list or tree items, move cards - between columns, drop into folders or zones, drag slider handles, draw on a canvas. - - Give visual descriptions of what to grab and where to drop it — the tool locates both points - on the screenshot and drags between them with a real mouse. When items look alike, describe - the position (e.g. "the second card in the first column"). For a copy-drag that duplicates - the item, pass modifier. - `, - inputSchema: z.object({ - from: z.string().describe('What to drag, by visible appearance and position (e.g. "card labeled Bug 5 in the first column", "round handle of the volume slider")'), - to: z.string().describe('Where to drop it, by visible appearance and position (e.g. "column labeled Done", "right end of the volume slider track")'), - modifier: z.enum(['Control', 'Shift', 'Alt', 'Meta', 'CommandOrControl']).optional().describe('Modifier key held during the whole drag, e.g. Control for a copy-drag that duplicates the item.'), - explanation: z.string().describe('Why you are dragging this element'), - }), - execute: async ({ from, to, modifier, explanation }) => { - const activeNote = task.startNote(explanation); - - if (!researcher || Stats.visionDisabled) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', 'Drag points cannot be located in this session. Use form() with I.dragAndDrop(source, target) instead.'); - } - - try { - const captured = await explorer.capture({ screenshot: true }); - - if (!captured.screenshot) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', 'Failed to capture screenshot for locating the drag points'); - } - - const analysis = await researcher.checkDragPoints(captured, from, to); - - if (!analysis) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', 'Visual analysis failed to process the screenshot'); - } - - const points = dragPoints(analysis); - - if (!points) { - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', `Drag points not found: ${analysis}`, { - analysis, - suggestion: 'Describe what to drag and where to drop it more precisely, or scroll the target into view and retry.', - }); - } - - const previousState = ActionResult.fromState(stateManager.getCurrentState()!); - const action = explorer.action(); - const dragCode = mouseDragCode(from, to, points); - const block = withModifier(dragCode, modifier); - const attempts: Array<{ command: string; success: boolean; error?: string }> = []; - - const mouseSuccess = await action.attempt(block, explanation); - const attempt: { command: string; success: boolean; error?: string } = { command: dragCode, success: mouseSuccess }; - if (!mouseSuccess && action.lastError) attempt.error = errorText(action.lastError); - attempts.push(attempt); - - if (mouseSuccess) { - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); - - if (hasObservablePageChange(toolResult)) { - await commitNote(activeNote, TestResult.PASSED, toolResult, action); - return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block }, action); - } - } - await releaseModifier(action, modifier); - - const dragged = await syntheticDragDrop(explorer, points, modifier); - attempts.push({ command: `${dragCode} (HTML5 drag events)`, success: dragged }); - - if (dragged) { - await explorer.capture(); - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); - - if (hasObservablePageChange(toolResult)) { - await commitNote(activeNote, TestResult.PASSED, toolResult, action); - return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block, message: 'The drop landed only after HTML5 drag events were dispatched at the located points; the recorded mouse drag replays through the tool, which runs the same escalation.' }, action); - } - } - - const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); - await commitNote(activeNote, TestResult.FAILED, toolResult, action); - return failedToolResult( - 'dragAndDrop', - 'The drag changed nothing on the page', - { - ...toolResult, - points, - analysis, - attempts, - suggestion: 'Check that what you described is draggable and the target is a valid drop zone. When a drag starts from a handle, describe the handle as the grab point, not the whole card.', - }, - action.lastError, - judge, - explanation - ); - } catch (error) { - throwIfFatalBrowserError(error); - activeNote.commit(TestResult.FAILED); - return failedToolResult('dragAndDrop', `dragAndDrop tool failed: ${errorText(error)}`); - } - }, - }), - pressKey: tool({ description: dedent` Press a keyboard key or key combination. Use this for special keys like Enter, Escape, Tab, Arrow keys, or key combinations with modifiers. @@ -1318,7 +1211,7 @@ function cap(text: string | undefined | null, max: number): string { return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`; } -function throwIfFatalBrowserError(error: unknown): void { +export function throwIfFatalBrowserError(error: unknown): void { if (isFatalBrowserError(error)) throw error; } @@ -1347,67 +1240,11 @@ function transformContainsCommand(command: string): string { return command; } -function errorText(error: unknown): string { +export function errorText(error: unknown): string { if (error instanceof Error) return compactErrorMessage(error); return 'Unknown error occurred'; } -function withModifier(command: string, modifier?: string): string { - if (!modifier) return command; - return `I.pressKeyDown(${JSON.stringify(modifier)})\n${command}\nI.pressKeyUp(${JSON.stringify(modifier)})`; -} - -async function releaseModifier(action: any, modifier?: string): Promise { - if (!modifier) return; - await action.attempt(`I.pressKeyUp(${JSON.stringify(modifier)})`, `Release the ${modifier} key left held by a failed drag`); -} - -const SYNTHETIC_MODIFIER_KEYS: Record = { Control: 'ctrlKey', Shift: 'shiftKey', Alt: 'altKey', Meta: 'metaKey', CommandOrControl: 'auto' }; - -export function dragPoints(analysis: string): DragPoints | null { - const match = analysis.match(/grab\s+(\d+)X,\s*(\d+)Y.*?drop\s+(\d+)X,\s*(\d+)Y/is); - if (!match) return null; - return { - from: { x: Number.parseInt(match[1], 10), y: Number.parseInt(match[2], 10) }, - to: { x: Number.parseInt(match[3], 10), y: Number.parseInt(match[4], 10) }, - }; -} - -function mouseDragCode(from: string, to: string, points: DragPoints): string { - const { x: fx, y: fy } = points.from; - const { x: tx, y: ty } = points.to; - return `I.usePlaywrightTo(${JSON.stringify(`drag ${from} to ${to}`)}, async ({ page }) => { await page.mouse.move(${fx}, ${fy}); await page.mouse.down(); await page.waitForTimeout(200); await page.mouse.move(${tx}, ${ty}, { steps: 10 }); await page.waitForTimeout(200); await page.mouse.up(); })`; -} - -export async function syntheticDragDrop(explorer: any, points: DragPoints, modifier?: string): Promise { - const modifierKey = modifier ? SYNTHETIC_MODIFIER_KEYS[modifier] : undefined; - return explorer - .withPage(async (page: any) => - page.evaluate( - ({ from, to, modifierKey }) => { - const startEl = document.elementFromPoint(from.x, from.y); - const endEl = document.elementFromPoint(to.x, to.y); - if (!startEl || !endEl) return false; - const key = modifierKey === 'auto' ? (navigator.platform.includes('Mac') ? 'metaKey' : 'ctrlKey') : modifierKey; - const dataTransfer = new DataTransfer(); - const fire = (el: Element, type: string, at: { x: number; y: number }) => { - const init: any = { bubbles: true, cancelable: true, dataTransfer, clientX: at.x, clientY: at.y }; - if (key) init[key] = true; - el.dispatchEvent(new DragEvent(type, init)); - }; - fire(startEl, 'dragstart', from); - fire(endEl, 'dragenter', to); - fire(endEl, 'dragover', to); - fire(endEl, 'drop', to); - fire(startEl, 'dragend', from); - return true; - }, - { from: points.from, to: points.to, modifierKey } - ) - ) - .catch(() => false); -} - export async function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise { if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) { activeNote.screenshot = await action.saveScreenshot(); @@ -1467,7 +1304,7 @@ export function formatExecutedSteps(steps: ExecutedStep[], requestedCount = step return lines.join('\n'); } -function hasObservablePageChange(data?: Record): boolean { +export function hasObservablePageChange(data?: Record): boolean { if (!data?.pageDiff) return false; if (data.pageDiff.urlChanged === true) return true; if (data.pageDiff.ariaChanges) return true; @@ -1631,8 +1468,3 @@ interface MatchedElement { label: string; visible?: boolean; } - -export interface DragPoints { - from: { x: number; y: number }; - to: { x: number; y: number }; -} diff --git a/src/explorbot.ts b/src/explorbot.ts index a07528ab..32b94a5d 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -318,7 +318,7 @@ export class ExplorBot { const researcher = this.agentResearcher(); const navigator = this.agentNavigator(); const tools = createAgentTools({ ...deps, researcher, navigator, withExperience: false }); - return new Rerunner(deps, tools, researcher); + return new Rerunner(deps, tools); }); if (this.isHistorianEnabled()) this.agents.rerunner.setHistorian(this.agentHistorian()); } @@ -328,7 +328,7 @@ export class ExplorBot { agentDriller(): Driller { return (this.agents.driller ||= this.createAgent((deps) => { const navigator = this.agentNavigator(); - return new Driller(deps, navigator, this.agentResearcher()); + return new Driller(deps, navigator); })); } diff --git a/src/test-plan.ts b/src/test-plan.ts index b0e5a2be..34580e14 100644 --- a/src/test-plan.ts +++ b/src/test-plan.ts @@ -259,6 +259,7 @@ export class Test extends Task { artifacts: Record; generatedCode?: string; style?: string; + explicitTools: string[] = []; planIteration = 0; enabled = true; startTime?: number; diff --git a/src/utils/test-plan-markdown.ts b/src/utils/test-plan-markdown.ts index 5181bd05..c31445c1 100644 --- a/src/utils/test-plan-markdown.ts +++ b/src/utils/test-plan-markdown.ts @@ -132,6 +132,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { let inSteps = false; let inExpected = false; let priority: 'critical' | 'important' | 'high' | 'normal' | 'low' = 'normal'; + let explicitTools: string[] = []; for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); @@ -142,6 +143,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { inSteps = false; inExpected = false; priority = 'normal'; + explicitTools = []; const title = lines[i + 1]?.replace(/^#\s+/, '') || ''; currentPlan = new Plan(title); plans.push(currentPlan); @@ -161,6 +163,11 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { } const priorityMatch = block.match(/priority:\s*(\w+)/); priority = (priorityMatch?.[1] as 'critical' | 'important' | 'high' | 'normal' | 'low') || 'normal'; + const toolsMatch = block.match(/tools:\s*(.+)/); + explicitTools = (toolsMatch?.[1] || '') + .split(',') + .map((name) => name.trim().toLowerCase()) + .filter(Boolean); i = j; continue; } @@ -168,6 +175,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { if (line.startsWith('# ') && currentTest === null) { const scenario = line.replace(/^#\s+/, ''); currentTest = new Test(scenario, priority, [], ''); + currentTest.explicitTools = explicitTools; currentPlan.addTest(currentTest); inRequirements = false; inSteps = false; @@ -274,7 +282,9 @@ function formatPlanSuite(plan: Plan): string { } for (const test of plan.tests) { - content += `\n`; + let testHeader = `\n`; content += `# ${test.scenario}\n\n`; content += '## Requirements\n'; content += `${test.startUrl || CURRENT_PAGE}\n\n`; diff --git a/tests/integration/drag-and-drop-browser.test.ts b/tests/integration/drag-and-drop-browser.test.ts index 4b4c1881..3f66bd50 100644 --- a/tests/integration/drag-and-drop-browser.test.ts +++ b/tests/integration/drag-and-drop-browser.test.ts @@ -1,7 +1,7 @@ import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; import { join } from 'node:path'; import { type Browser, type Page, chromium } from 'playwright'; -import { type DragPoints, syntheticDragDrop } from '../../src/ai/tools.ts'; +import { type DragPoints, syntheticDragDrop } from '../../src/ai/explicit-tools.ts'; let browser: Browser; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 6e5017d2..cd969dd9 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -152,6 +152,7 @@ describe('Planner with aimock', () => { clearPlanRegistry(); clearSessionDedup(); clearStyleCache(); + provider.setJudge({ assert: async () => false } as any); planner = new Planner({ ...createMockDeps(), ai: provider } as any, { research: async () => taskBoardUiMap } as any); @@ -563,4 +564,20 @@ describe('Planner with aimock', () => { expect(prompt).toContain('page or subpage has not been observed'); expect(prompt).toContain('describe the expected visible result generically'); }); + + it('arms explicit tools when the decision model says the scenario needs them', async () => { + provider.setJudge({ assert: async () => true } as any); + + const plan = await planner.plan(); + + expect(plan.tests[0].explicitTools).toEqual(['dragAndDrop', 'rightClick']); + }); + + it('keeps explicit tools off when the decision model declines', async () => { + provider.setJudge({ assert: async () => false } as any); + + const plan = await planner.plan(); + + expect(plan.tests.every((t) => t.explicitTools.length === 0)).toBe(true); + }); }); diff --git a/tests/unit/drag-and-drop.test.ts b/tests/unit/drag-and-drop.test.ts index b482d06e..160c1d86 100644 --- a/tests/unit/drag-and-drop.test.ts +++ b/tests/unit/drag-and-drop.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it } from 'bun:test'; -import { createCodeceptJSTools, dragPoints } from '../../src/ai/tools.ts'; +import { createExplicitTools, dragPoints } from '../../src/ai/explicit-tools.ts'; +import { createCodeceptJSTools } from '../../src/ai/tools.ts'; import { ConfigParser } from '../../src/config.ts'; const ANALYSIS = 'The first card and the Done column are visible. grab 10X, 20Y; drop 30X, 40Y'; @@ -52,6 +53,11 @@ describe('dragPoints parsing', () => { }); }); + it('keeps dragAndDrop out of the default CodeceptJS tools', () => { + const tools = createCodeceptJSTools(fakeDeps().deps as any, fakeTask()); + expect(tools.dragAndDrop).toBeUndefined(); + }); + it('reads the points from a sentence', () => { expect(dragPoints('The card is in the first column. grab 5X, 6Y; drop 7X, 8Y')).toEqual({ from: { x: 5, y: 6 }, @@ -74,7 +80,7 @@ describe('dragAndDrop tool', () => { it('drags between the located points with a real mouse and records replayable code', async () => { const { deps, action, holder } = fakeDeps(); succeedWithReorder(action, holder); - const tools = createCodeceptJSTools(deps, fakeTask()); + const tools = createExplicitTools(deps, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); @@ -90,7 +96,7 @@ describe('dragAndDrop tool', () => { it('wraps the drag between pressKeyDown and pressKeyUp for a modifier', async () => { const { deps, action, holder } = fakeDeps(); succeedWithReorder(action, holder); - const tools = createCodeceptJSTools(deps, fakeTask()); + const tools = createExplicitTools(deps, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); @@ -103,7 +109,7 @@ describe('dragAndDrop tool', () => { it('fails without browser action when vision cannot locate both points', async () => { const { deps, action } = fakeDeps(); deps.researcher.checkDragPoints = async () => 'The drop target was not found'; - const tools = createCodeceptJSTools(deps, fakeTask()); + const tools = createExplicitTools(deps, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); @@ -132,7 +138,7 @@ describe('dragAndDrop tool', () => { return true; }, }; - const tools = createCodeceptJSTools(deps, fakeTask()); + const tools = createExplicitTools(deps, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); @@ -149,7 +155,7 @@ describe('dragAndDrop tool', () => { return command.startsWith('I.pressKeyUp'); }; deps.explorer.withPage = async () => false; - const tools = createCodeceptJSTools(deps, fakeTask()); + const tools = createExplicitTools(deps, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); @@ -160,7 +166,7 @@ describe('dragAndDrop tool', () => { it('reports drag as unavailable when there is no researcher', async () => { const { deps, action } = fakeDeps(); - const tools = createCodeceptJSTools({ ...deps, researcher: undefined }, fakeTask()); + const tools = createExplicitTools({ ...deps, researcher: undefined }, fakeTask()); const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); diff --git a/tests/unit/explicit-tools.test.ts b/tests/unit/explicit-tools.test.ts new file mode 100644 index 00000000..7f1b7e10 --- /dev/null +++ b/tests/unit/explicit-tools.test.ts @@ -0,0 +1,200 @@ +import { beforeEach, describe, expect, it } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { EXPLICIT_TOOL_NAMES, createExplicitTools, parseExplicitToolsRequest } from '../../src/ai/explicit-tools.ts'; +import { Tester } from '../../src/ai/tester.ts'; +import { ConfigParser } from '../../src/config.ts'; +import { Plan, Test } from '../../src/test-plan.ts'; +import { parsePlansFromMarkdownText, savePlanToMarkdown } from '../../src/utils/test-plan-markdown.ts'; + +function fakeDeps() { + const holder: { state: any } = { + state: { url: '/board', html: '', ariaSnapshot: '- listitem "First"', id: 'before' }, + }; + const action: any = { + lastError: null, + executedSteps: [] as Array<{ command: string; success: boolean }>, + ran: [] as string[], + saveScreenshot: async () => undefined, + attempt: async (command: string) => { + action.ran.push(command); + return false; + }, + }; + const deps: any = { + explorer: { action: () => action, capture: async () => ({ screenshot: Buffer.from('png') }) }, + stateManager: { getCurrentState: () => holder.state }, + ai: {}, + }; + return { deps, action, holder }; +} + +function fakeTask(): any { + return { startNote: () => ({ commit: () => {}, screenshot: undefined }) }; +} + +function openMenu(action: any, holder: { state: any }) { + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = [{ command, success: true }]; + holder.state = { ...holder.state, ariaSnapshot: '- menu "Rename"\n- menu "Delete"', id: 'after' }; + return true; + }; +} + +describe('rightClick tool', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + it('right-clicks the first working locator and reports the opened menu', async () => { + const { deps, action, holder } = fakeDeps(); + openMenu(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5", ".board")', 'I.rightClick(".card-5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(true); + expect(action.ran).toEqual(['I.rightClick("Bug 5", ".board")']); + expect(result.code).toBe('I.rightClick("Bug 5", ".board")'); + }); + + it('wraps a bare locator into I.rightClick()', async () => { + const { deps, action, holder } = fakeDeps(); + openMenu(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['Bug 5'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(true); + expect(action.ran).toEqual(['I.rightClick("Bug 5")']); + }); + + it('rejects non-right-click commands without touching the browser', async () => { + const { deps, action } = fakeDeps(); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.click("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('only accepts I.rightClick()'); + expect(action.ran).toEqual([]); + }); + + it('fails when all locators fail', async () => { + const { deps, action } = fakeDeps(); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('All right-click commands failed'); + expect(action.ran).toEqual(['I.rightClick("Bug 5")']); + }); + + it('fails when the click lands but no context menu appears', async () => { + const { deps, action } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = [{ command, success: true }]; + return true; + }; + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('no context menu appeared'); + }); +}); + +describe('parseExplicitToolsRequest', () => { + it('reads requested tool names from pilot guidance', () => { + expect(parseExplicitToolsRequest('Move the card.\nENABLE_TOOLS: dragAndDrop, rightClick\nFollow this plan.')).toEqual(['dragAndDrop', 'rightClick']); + }); + + it('ignores unknown names and is case-insensitive', () => { + expect(parseExplicitToolsRequest('ENABLE_TOOLS: draganddrop, hover, click')).toEqual(['dragAndDrop']); + }); + + it('returns nothing without the marker', () => { + expect(parseExplicitToolsRequest('Use click() on the card.')).toEqual([]); + }); +}); + +describe('explicit tools in the test plan', () => { + let tmpPath: string; + + beforeEach(() => { + tmpPath = mkdtempSync(join(tmpdir(), 'explorbot-explicit-')); + }); + + it('round-trips the tools flag through the plan markdown', () => { + const plan = new Plan('/board'); + const dragTest = new Test('Reorder cards by dragging', 'high', ['cards are reordered'], '/board', ['drag the card']); + dragTest.explicitTools = [...EXPLICIT_TOOL_NAMES]; + const plainTest = new Test('Open the board', 'normal', ['board is visible'], '/board', []); + plan.addTest(dragTest); + plan.addTest(plainTest); + const file = join(tmpPath, 'plan.md'); + savePlanToMarkdown(plan, file); + + const restored = parsePlansFromMarkdownText(readFileSync(file, 'utf-8'))[0]; + + expect(restored.tests[0].explicitTools).toEqual(['draganddrop', 'rightclick']); + expect(restored.tests[1].explicitTools).toEqual([]); + }); + + it('defaults to no explicit tools for plans written without the flag', () => { + const content = ['', '# /board', '', '', '# Reorder cards', '', '## Requirements', '', '/board', ''].join('\n'); + const restored = parsePlansFromMarkdownText(content)[0]; + + expect(restored.tests[0].explicitTools).toEqual([]); + }); +}); + +describe('tester explicit tools gating', () => { + function buildTester() { + const deps: any = { + explorer: {}, + ai: { getSystemPromptForAgent: () => '' }, + config: {}, + stateManager: { + getCurrentState: () => null, + otherTabs: [], + getExperienceTracker: () => ({ getExperienceTableOfContents: () => [] }), + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', renderRelevantContext: () => '' }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + return new Tester(deps, {} as any, {} as any); + } + + it('keeps the explicit tools rule out of the system message by default', () => { + const tester = buildTester(); + + expect(tester.getSystemMessage()).not.toContain(''); + }); + + it('injects the rule when the plan armed the tools', () => { + const tester = buildTester(); + (tester as any).enabledExplicitTools = new Set(['dragAndDrop']); + + expect(tester.getSystemMessage()).toContain('### dragAndDrop() tool'); + }); + + it('enables tools from the pilot marker and returns the rules to inject', () => { + const tester = buildTester(); + const rules = (tester as any).applyExplicitTools('ENABLE_TOOLS: dragAndDrop, rightClick'); + + expect((tester as any).enabledExplicitTools).toEqual(new Set(['dragAndDrop', 'rightClick'])); + expect(rules).toContain(''); + expect((tester as any).applyExplicitTools('ENABLE_TOOLS: dragAndDrop')).toBe(''); + expect((tester as any).applyExplicitTools('No marker here')).toBe(''); + }); +});