diff --git a/CHANGELOG.md b/CHANGELOG.md index 901a0a2d..b1678207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -238,6 +238,20 @@ - An `explore` run ends with an "Explore more" section suggesting `explorbot explore /` to cover the whole site, plus a link to the docs. - Init makes it explicit how to connect the AI: the `.env` template and the missing-key warning point to the provider's key page (e.g. `https://openrouter.ai/keys`) and show the exact line to paste — `OPENROUTER_API_KEY=`; the setup wizard shows where to create the key too. +## 2026-11-09 + +### Changes +- [Tester] Non-standard interaction tools — `dragAndDrop()` and `rightClick()` — are now explicit tools: off by default, so they never grow the Tester's context for ordinary scenarios. The Planner decides per scenario (via the decision model) whether a drag or a context menu is needed and arms the tools for exactly those scenarios; the choice is stored in the plan file (`tools:` line in the `` comment) and survives re-runs. Mid-run, the Pilot can also arm the tools by including `ENABLE_TOOLS: ` in its instruction when a scenario turns out to need an interaction `click()`/`form()` cannot express. + +## 2026-10-08 + +### Changes +- [Tester] New `dragAndDrop()` tool, so scenarios that drag something onto something else can be tested: reordering lists and trees, moving cards between columns, dropping items into folders or zones — and dragging slider handles or drawing on canvas, where nothing can be addressed by a locator. The tool takes visual descriptions of what to grab and where to drop it, the vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API. When the drop changes nothing, the same points are retried with HTML5 drag events — covering pages whose drag libraries cancel the mouse press. + ```javascript + dragAndDrop({ from: 'card labeled Bug 5 in the first column', to: 'column labeled Done' }); // move or reorder + ``` + Copy-drag: pass the modifier (e.g. `Control`) to the tool; the generated test records the drag as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence — so it replays and heals in rerun like any other step. + ## 2026-10-06 ### Configuration @@ -251,8 +265,8 @@ - OpenRouter's recommended agentic model is now Claude Haiku 5.5 (`anthropic/claude-haiku-5.5`). - OpenAI's recommended vision and agentic model is now GPT-6 Luna (`gpt-6-luna`). - `explorbot recommended-models` lists the decision model and its `EXPLORBOT_DECISION_MODEL` env line for providers that recommend one. -### Changes - A `url` in the config file can now point to an app in a subfolder, such as `https://example.com/apps/todo/`. The folder becomes the base path for the run, just like `--base-url`: `explorbot start` opens the app itself, and relative paths such as `/active` resolve to `/apps/todo/active`. Previously every navigation was built as `https://example.com/apps/todo///…` and the Navigator had to guess its way back, sometimes ending on the wrong page. `--base-url` still takes precedence over the config `url`. +- Generated tests no longer break on a locator that contains quotes, such as `li:has-text("Item 2")`: executed commands are recorded with the quotes escaped, so the test file stays valid code and reruns instead of failing to load. ## 2026-10-05 diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 9aedbdca..e971689e 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -11,11 +11,11 @@ import { createDocsCommands } from '../boat/doc-collector/src/cli.ts'; import { createPrimaCommands } from '../boat/prima/src/cli.ts'; import { flushTelemetry } from '../src/ai/provider.js'; import { HelpJsonCommand } from '../src/commands/help-json-command.js'; -import { RecommendedModelsCommand } from '../src/commands/recommended-models-command.js'; +import { authOption, decisionModelOption, knowledgeOption, wsOption } from '../src/commands/options/index.js'; import { PlanCommand } from '../src/commands/plan-command.js'; +import { RecommendedModelsCommand } from '../src/commands/recommended-models-command.js'; import { App } from '../src/components/App.js'; import { StatusPane } from '../src/components/StatusPane.js'; -import { authOption, decisionModelOption, knowledgeOption, wsOption } from '../src/commands/options/index.js'; import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS, outputPath } from '../src/config.js'; import { ExplorBot, type ExplorBotOptions } from '../src/explorbot.js'; import { remote } from '../src/remote.js'; diff --git a/boat/doc-collector/src/ai/documentarian.ts b/boat/doc-collector/src/ai/documentarian.ts index 8af1b323..58b68946 100644 --- a/boat/doc-collector/src/ai/documentarian.ts +++ b/boat/doc-collector/src/ai/documentarian.ts @@ -1,12 +1,12 @@ import dedent from 'dedent'; import { z } from 'zod'; import type { AIProvider } from '../../../../src/ai/provider.ts'; +import { parseResearchSections } from '../../../../src/ai/researcher/parser.ts'; import type Explorer from '../../../../src/explorer.ts'; import type { StateManager } from '../../../../src/state-manager.ts'; import type { WebPageState } from '../../../../src/state-manager.ts'; import { parseAriaLocator } from '../../../../src/utils/aria.ts'; import { tag } from '../../../../src/utils/logger.ts'; -import { parseResearchSections } from '../../../../src/ai/researcher/parser.ts'; import type { DocbotConfig } from '../config.ts'; import { type CaptureInteractionState, type DocStateTransition, collectDocInteractions } from './tools.ts'; diff --git a/boat/doc-collector/src/docbot.ts b/boat/doc-collector/src/docbot.ts index aed8f5cd..108f3178 100644 --- a/boat/doc-collector/src/docbot.ts +++ b/boat/doc-collector/src/docbot.ts @@ -4,8 +4,8 @@ import { ExplorBot, type ExplorBotOptions } from '../../../src/explorbot.ts'; import type { Link, WebPageState } from '../../../src/state-manager.ts'; import { normalizeUrl } from '../../../src/state-manager.ts'; import { tag } from '../../../src/utils/logger.ts'; -import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { keepSiteUrls } from '../../../src/utils/sitemap.ts'; +import { sanitizeFilename } from '../../../src/utils/strings.ts'; import { Documentarian, type PageDocumentation } from './ai/documentarian.ts'; import type { DocStateTransition } from './ai/tools.ts'; import { type DocbotConfig, DocbotConfigParser } from './config.ts'; diff --git a/boat/doc-collector/src/path-filter.ts b/boat/doc-collector/src/path-filter.ts index a2fda2c0..74b77c5f 100644 --- a/boat/doc-collector/src/path-filter.ts +++ b/boat/doc-collector/src/path-filter.ts @@ -1,5 +1,5 @@ import { normalizeUrl } from '../../../src/state-manager.ts'; -import { matchesUrl, generalizeUrl } from '../../../src/utils/url-matcher.ts'; +import { generalizeUrl, matchesUrl } from '../../../src/utils/url-matcher.ts'; import type { DocbotConfig } from './config.ts'; const DEFAULT_DENIED_PATH_SEGMENTS = ['callback', 'callbacks', 'logout', 'signout', 'sign_out', 'destroy', 'delete', 'remove']; diff --git a/boat/doc-collector/src/research-navigation.ts b/boat/doc-collector/src/research-navigation.ts index e130a0d2..ae376588 100644 --- a/boat/doc-collector/src/research-navigation.ts +++ b/boat/doc-collector/src/research-navigation.ts @@ -1,5 +1,5 @@ +import { type ResearchElement, parseResearchSections } from '../../../src/ai/researcher/parser.ts'; import type { WebPageState } from '../../../src/state-manager.ts'; -import { parseResearchSections, type ResearchElement } from '../../../src/ai/researcher/parser.ts'; const OPEN_API_TAG_SELECTOR_PATTERN = /api-\d+\/tag\/([a-z0-9-]+)(?:["'#/\]\s]|$)/i; const OPEN_API_NAVIGATION_SECTION_KEYWORDS = ['navigation', 'menu']; diff --git a/boat/doc-collector/src/screenshots.ts b/boat/doc-collector/src/screenshots.ts index 3f26fa12..72033cdd 100644 --- a/boat/doc-collector/src/screenshots.ts +++ b/boat/doc-collector/src/screenshots.ts @@ -3,9 +3,9 @@ import path from 'node:path'; import { PNG } from 'pngjs'; import { removeVisualAnnotations } from '../../../src/ai/researcher/coordinates.ts'; import { parseResearchSections } from '../../../src/ai/researcher/parser.ts'; -import { parseAriaLocator } from '../../../src/utils/aria.ts'; import type Explorer from '../../../src/explorer.ts'; import { type WebPageState, normalizeUrl } from '../../../src/state-manager.ts'; +import { parseAriaLocator } from '../../../src/utils/aria.ts'; import { safeFilename, sanitizeFilename } from '../../../src/utils/strings.ts'; import type { PageDocumentation } from './ai/documentarian.ts'; import type { DocStateTransition } from './ai/tools.ts'; diff --git a/docs/web-testing/agents.md b/docs/web-testing/agents.md index a26ce6ae..4b0fdc24 100644 --- a/docs/web-testing/agents.md +++ b/docs/web-testing/agents.md @@ -16,7 +16,7 @@ flowchart LR ## Navigator Agent -Handles browser interactions: clicks, form fills, and navigation. +Handles browser interactions: clicks, drag and drop, form fills, and navigation. The Navigator runs CodeceptJS commands in the browser. When a selector fails, it tries other locator strategies and resolves the interaction without stopping the run. It remembers what worked and what didn't, so failed selectors don't keep killing your runs and tests survive UI changes. diff --git a/docs/web-testing/page-interaction.md b/docs/web-testing/page-interaction.md index 9c30663c..67dc88b2 100644 --- a/docs/web-testing/page-interaction.md +++ b/docs/web-testing/page-interaction.md @@ -72,6 +72,15 @@ The agent starts with short semantic locators from ARIA or visible text. When a When a locator fails, the agent tries the next strategy, then a visual click. Locators that worked are saved to experience and preferred on the next run. +## Explicit tools: drag and drop, right-click + +Some interactions — dragging an element, opening a context menu with a right-click — need dedicated tools (`dragAndDrop`, `rightClick` in `src/ai/explicit-tools.ts`). They are **off by default**, so they never grow the Tester's context for ordinary scenarios. Two gates turn them on: + +- **Planner gate** — during planning, each scenario is checked with the decision model (`provider.oracle()`, with the agentic model as fallback). A scenario that needs a drag or a context menu gets the tools armed; the choice is stored in the plan (`tools:` in the `` comment) and survives re-runs. +- **Pilot gate** — mid-run, the Pilot can request the tools for the Tester by including `ENABLE_TOOLS: ` in its instruction, when the scenario turns out to need an interaction `click()`/`form()` cannot express. + +The `dragAndDrop()` tool takes visual descriptions of what to grab and where to drop it. The vision model locates both points on the screenshot, and the drag runs as a real mouse press between them through the Playwright API — which also covers sliders and canvas drawing, where there is nothing to attach a locator to. When the drop changes nothing, the tool retries the same points with HTML5 drag events on its own; that covers pages whose drag libraries cancel the mouse press, so a real mouse can never start the drag. A modifier key can be held during the drag for copy-drags. The drag is recorded as a plain CodeceptJS `I.usePlaywrightTo` mouse sequence, so generated tests replay unchanged. + ## What happens after each action After every action, Explorbot captures the new page state and compares it with the previous one. The resulting diff tells the agent what changed — the URL, the accessibility tree, or the HTML — so it can confirm the action worked and decide what to do next. The Researcher turns a page into a structured UI map of sections and elements; see [Researcher](./researcher.md). diff --git a/src/action.ts b/src/action.ts index f6f4b775..db9fb6a4 100644 --- a/src/action.ts +++ b/src/action.ts @@ -583,11 +583,7 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra } if (batchFailed) return; - const executed: ExecutedStep = { command: step.toCode(), success: !error }; - if (Object.keys(step.opts || {}).length) { - const args = [step.humanizeArgs(), `step.opts(${JSON.stringify(step.opts)})`].filter(Boolean).join(', '); - executed.command = `${step.prefix}${step.actor}.${step.title}(${args})${step.suffix}`; - } + const executed: ExecutedStep = { command: stepCode(step), success: !error }; if (error) { executed.error = errorToString(error); batchFailed = true; @@ -612,6 +608,24 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra }; }; +function stepCode(step: any): string { + const needsEscaping = (step.args || []).some((arg: any) => typeof arg === 'string' && arg.includes('"')); + if (!needsEscaping) { + if (!Object.keys(step.opts || {}).length) return step.toCode(); + const args = [step.humanizeArgs(), `step.opts(${JSON.stringify(step.opts)})`].filter(Boolean).join(', '); + return `${step.prefix}${step.actor}.${step.title}(${args})${step.suffix}`; + } + const args = (step.args || []) + .filter((arg: any) => arg !== undefined) + .map((arg: any) => { + if (typeof arg === 'function') return arg.toString(); + if (arg && typeof arg.getMasked === 'function') return JSON.stringify(arg.getMasked()); + return JSON.stringify(arg); + }); + if (Object.keys(step.opts || {}).length) args.push(`step.opts(${JSON.stringify(step.opts)})`); + return `${step.prefix}${step.actor}.${step.name}(${args.join(', ')})${step.suffix}`; +} + const readFocusedElement = () => { const el = document.activeElement as any; if (!el || el === document.body) return null; diff --git a/src/ai/captain/idle-mode.ts b/src/ai/captain/idle-mode.ts index c9e22aee..99d92cfb 100644 --- a/src/ai/captain/idle-mode.ts +++ b/src/ai/captain/idle-mode.ts @@ -4,8 +4,8 @@ import dedent from 'dedent'; import { z } from 'zod'; import { ConfigParser } from '../../config.ts'; import { Plan, Test } from '../../test-plan.ts'; -import { getStyles } from '../planner/styles.ts'; import { PlanningBlockedError } from '../../utils/error-page.ts'; +import { getStyles } from '../planner/styles.ts'; import { listRecentArtifacts, readCaptainFile } from './file-tools.ts'; import { type Constructor, type ModeContext, resolveProjectRoot } from './mixin.ts'; diff --git a/src/ai/captain/web-mode.ts b/src/ai/captain/web-mode.ts index 4ae0f62f..c667f205 100644 --- a/src/ai/captain/web-mode.ts +++ b/src/ai/captain/web-mode.ts @@ -11,7 +11,7 @@ export function WithWebMode(Base: T) { webModeTools(ctx: ModeContext): Record { const explorer = ctx.explorBot.getExplorer(); const toolDeps = { explorer, stateManager: ctx.explorBot.stateManager(), ai: ctx.explorBot.getProvider() }; - const codeceptTools = createCodeceptJSTools(toolDeps, ctx.task); + const codeceptTools = createCodeceptJSTools({ ...toolDeps, researcher: ctx.explorBot.agentResearcher() }, ctx.task); const agentTools = createAgentTools({ ...toolDeps, researcher: ctx.explorBot.agentResearcher(), diff --git a/src/ai/explicit-tools.ts b/src/ai/explicit-tools.ts new file mode 100644 index 00000000..d2351f0c --- /dev/null +++ b/src/ai/explicit-tools.ts @@ -0,0 +1,284 @@ +import { tool } from 'ai'; +import dedent from 'dedent'; +import { z } from 'zod'; +import { ActionResult } from '../action-result.ts'; +import { Stats } from '../stats.ts'; +import { type Task, TestResult } from '../test-plan.js'; +import type { ToolDeps } from './agent.ts'; +import type { Researcher } from './researcher.ts'; +import { clickFailureSuggestion, commitNote, errorText, failedToolResult, hasObservablePageChange, successToolResult, throwIfFatalBrowserError } from './tools.ts'; + +export const EXPLICIT_TOOL_NAMES = ['dragAndDrop', 'rightClick'] as const; + +interface ExplicitToolDeps extends ToolDeps { + researcher?: Researcher; +} + +export function createExplicitTools({ explorer, stateManager, judge, researcher }: ExplicitToolDeps, task: Task): Record { + return { + dragAndDrop: tool({ + description: dedent` + Drag something on the page and drop it somewhere else: reorder list or tree items, move cards + between columns, drop into folders or zones, drag slider handles, draw on a canvas. + + Give visual descriptions of what to grab and where to drop it — the tool locates both points + on the screenshot and drags between them with a real mouse. When items look alike, describe + the position (e.g. "the second card in the first column"). For a copy-drag that duplicates + the item, pass modifier. + `, + inputSchema: z.object({ + from: z.string().describe('What to drag, by visible appearance and position (e.g. "card labeled Bug 5 in the first column", "round handle of the volume slider")'), + to: z.string().describe('Where to drop it, by visible appearance and position (e.g. "column labeled Done", "right end of the volume slider track")'), + modifier: z.enum(['Control', 'Shift', 'Alt', 'Meta', 'CommandOrControl']).optional().describe('Modifier key held during the whole drag, e.g. Control for a copy-drag that duplicates the item.'), + explanation: z.string().describe('Why you are dragging this element'), + }), + execute: async ({ from, to, modifier, explanation }) => { + const activeNote = task.startNote(explanation); + + if (!researcher || Stats.visionDisabled) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Drag points cannot be located in this session. Use form() with I.dragAndDrop(source, target) instead.'); + } + + try { + const captured = await explorer.capture({ screenshot: true }); + + if (!captured.screenshot) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Failed to capture screenshot for locating the drag points'); + } + + const analysis = await researcher.checkDragPoints(captured, from, to); + + if (!analysis) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', 'Visual analysis failed to process the screenshot'); + } + + const points = dragPoints(analysis); + + if (!points) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `Drag points not found: ${analysis}`, { + analysis, + suggestion: 'Describe what to drag and where to drop it more precisely, or scroll the target into view and retry.', + }); + } + + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const dragCode = mouseDragCode(from, to, points); + const block = withModifier(dragCode, modifier); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + + const mouseSuccess = await action.attempt(block, explanation); + const attempt: { command: string; success: boolean; error?: string } = { command: dragCode, success: mouseSuccess }; + if (!mouseSuccess && action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); + + if (mouseSuccess) { + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block }, action); + } + } + await releaseModifier(action, modifier); + + const dragged = await syntheticDragDrop(explorer, points, modifier); + attempts.push({ command: `${dragCode} (HTML5 drag events)`, success: dragged }); + + if (dragged) { + await explorer.capture(); + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + + if (hasObservablePageChange(toolResult)) { + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('dragAndDrop', { ...toolResult, points, analysis, attempts, code: block, message: 'The drop landed only after HTML5 drag events were dispatched at the located points; the recorded mouse drag replays through the tool, which runs the same escalation.' }, action); + } + } + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, dragCode); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'dragAndDrop', + 'The drag changed nothing on the page', + { + ...toolResult, + points, + analysis, + attempts, + suggestion: 'Check that what you described is draggable and the target is a valid drop zone. When a drag starts from a handle, describe the handle as the grab point, not the whole card.', + }, + action.lastError, + judge, + explanation + ); + } catch (error) { + throwIfFatalBrowserError(error); + activeNote.commit(TestResult.FAILED); + return failedToolResult('dragAndDrop', `dragAndDrop tool failed: ${errorText(error)}`); + } + }, + }), + + rightClick: tool({ + description: dedent` + Right-click an element to open its context menu, then act on the menu items it reveals. + + This tool ONLY accepts I.rightClick() commands. After the menu opens, click its items + with click() — the menu is not part of this tool. + `, + inputSchema: z.object({ + commands: z.array(z.string()).describe(dedent` + FALLBACK LOCATORS for ONE element. All commands must right-click the SAME element. + Order by reliability: + 1. I.rightClick(text, container) + 2. I.rightClick(ARIA, container) + 3. I.rightClick(CSS, container) + 4. I.rightClick(CSS) or I.rightClick(XPath) + `), + explanation: z.string().describe('Why you are right-clicking this element'), + }), + execute: async ({ commands: rawCommands, explanation }) => { + const activeNote = task.startNote(explanation); + + if (rawCommands.length === 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', 'No commands provided'); + } + + const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.rightClick')); + + if (invalidCommands.length > 0) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', `Invalid commands: ${invalidCommands.join(', ')}. rightClick tool only accepts I.rightClick() commands.`, { + suggestion: 'Use click() for normal clicks, or form() for multiple actions.', + }); + } + + const commands = rawCommands.map((cmd) => { + const trimmed = cmd.trim(); + if (trimmed.startsWith('I.rightClick')) return trimmed; + return `I.rightClick(${JSON.stringify(trimmed)})`; + }); + + const previousState = ActionResult.fromState(stateManager.getCurrentState()!); + const action = explorer.action(); + const attempts: Array<{ command: string; success: boolean; error?: string }> = []; + + for (const command of commands) { + const success = await action.attempt(command, explanation); + const attempt: { command: string; success: boolean; error?: string } = { command, success }; + if (action.lastError) attempt.error = errorText(action.lastError); + attempts.push(attempt); + + if (!success) continue; + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command); + + if (!hasObservablePageChange(toolResult)) { + activeNote.commit(TestResult.FAILED); + return failedToolResult('rightClick', 'Right-click executed, but no context menu appeared.', { + ...toolResult, + attempts, + code: command, + suggestion: 'Treat the element as not right-clickable. It may not attach a context menu, or the menu opens on hover instead.', + }); + } + + await commitNote(activeNote, TestResult.PASSED, toolResult, action); + return successToolResult('rightClick', { ...toolResult, attempts, code: command }, action); + } + + const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]); + await commitNote(activeNote, TestResult.FAILED, toolResult, action); + return failedToolResult( + 'rightClick', + 'All right-click commands failed', + { + ...toolResult, + attempts, + suggestion: clickFailureSuggestion(attempts), + }, + action.lastError, + judge, + explanation + ); + }, + }), + }; +} + +export function normalizeExplicitTools(names: string[]): string[] { + const canonical = names.map((name) => EXPLICIT_TOOL_NAMES.find((known) => known.toLowerCase() === name.trim().toLowerCase())).filter(Boolean) as string[]; + return [...new Set(canonical)]; +} + +export function parseExplicitToolsRequest(text: string): string[] { + const match = text.match(/^ENABLE_TOOLS:\s*([A-Za-z, ]+)$/m); + if (!match) return []; + return normalizeExplicitTools(match[1].split(',')); +} + +function withModifier(command: string, modifier?: string): string { + if (!modifier) return command; + return `I.pressKeyDown(${JSON.stringify(modifier)})\n${command}\nI.pressKeyUp(${JSON.stringify(modifier)})`; +} + +async function releaseModifier(action: any, modifier?: string): Promise { + if (!modifier) return; + await action.attempt(`I.pressKeyUp(${JSON.stringify(modifier)})`, `Release the ${modifier} key left held by a failed drag`); +} + +const SYNTHETIC_MODIFIER_KEYS: Record = { Control: 'ctrlKey', Shift: 'shiftKey', Alt: 'altKey', Meta: 'metaKey', CommandOrControl: 'auto' }; + +export function dragPoints(analysis: string): DragPoints | null { + const match = analysis.match(/grab\s+(\d+)X,\s*(\d+)Y.*?drop\s+(\d+)X,\s*(\d+)Y/is); + if (!match) return null; + return { + from: { x: Number.parseInt(match[1], 10), y: Number.parseInt(match[2], 10) }, + to: { x: Number.parseInt(match[3], 10), y: Number.parseInt(match[4], 10) }, + }; +} + +function mouseDragCode(from: string, to: string, points: DragPoints): string { + const { x: fx, y: fy } = points.from; + const { x: tx, y: ty } = points.to; + return `I.usePlaywrightTo(${JSON.stringify(`drag ${from} to ${to}`)}, async ({ page }) => { await page.mouse.move(${fx}, ${fy}); await page.mouse.down(); await page.waitForTimeout(200); await page.mouse.move(${tx}, ${ty}, { steps: 10 }); await page.waitForTimeout(200); await page.mouse.up(); })`; +} + +export async function syntheticDragDrop(explorer: any, points: DragPoints, modifier?: string): Promise { + const modifierKey = modifier ? SYNTHETIC_MODIFIER_KEYS[modifier] : undefined; + return explorer + .withPage(async (page: any) => + page.evaluate( + ({ from, to, modifierKey }: { from: { x: number; y: number }; to: { x: number; y: number }; modifierKey?: string }) => { + const startEl = document.elementFromPoint(from.x, from.y); + const endEl = document.elementFromPoint(to.x, to.y); + if (!startEl || !endEl) return false; + const key = modifierKey === 'auto' ? (navigator.platform.includes('Mac') ? 'metaKey' : 'ctrlKey') : modifierKey; + const dataTransfer = new DataTransfer(); + const fire = (el: Element, type: string, at: { x: number; y: number }) => { + const init: any = { bubbles: true, cancelable: true, dataTransfer, clientX: at.x, clientY: at.y }; + if (key) init[key] = true; + el.dispatchEvent(new DragEvent(type, init)); + }; + fire(startEl, 'dragstart', from); + fire(endEl, 'dragenter', to); + fire(endEl, 'dragover', to); + fire(endEl, 'drop', to); + fire(startEl, 'dragend', from); + return true; + }, + { from: points.from, to: points.to, modifierKey } + ) + ) + .catch(() => false); +} + +export interface DragPoints { + from: { x: number; y: number }; + to: { x: number; y: number }; +} diff --git a/src/ai/judge.ts b/src/ai/judge.ts index 6f93017a..5c125663 100644 --- a/src/ai/judge.ts +++ b/src/ai/judge.ts @@ -1,5 +1,5 @@ -import { clearActivity, setActivity } from '../activity.ts'; import type { ActionResult } from '../action-result.ts'; +import { clearActivity, setActivity } from '../activity.ts'; import type { AIConfig } from '../config.ts'; import { Observability } from '../observability.ts'; import { createDebug, tag } from '../utils/logger.ts'; diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 2fcde8fa..26f8c721 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -19,8 +19,8 @@ import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; import { normalizeInlineText } from '../utils/strings.ts'; -import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { UrlFilter } from '../utils/url-filter.ts'; +import { extractStatePath, isSameHostFamily, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { Agent, AgentDeps } from './agent.js'; import { type Decision, JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts'; import type { Provider } from './provider.js'; diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index aec4c923..e4a9b0bf 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1264,8 +1264,13 @@ export class Pilot implements Agent { To request more context, mention ATTACH_ARIA, ATTACH_SUMMARY, or ATTACH_UI_MAP — only when recent actions show failures. - Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck, - visualClick, back, getVisitedStates, reset, stop, finish, record. + dragAndDrop and rightClick are extra tools: Tester has them only when the plan enables them or after you request them. + Request by including a line "ENABLE_TOOLS: " in your instruction — when the scenario needs + dragging an element or opening a context menu and click()/form() cannot achieve it (for example, repeated failed + attempts to move an item). + + Tester tools: click, hover, pressKey, form, see, verify, interact, context, research, xpathCheck, + visualClick, back, getVisitedStates, reset, stop, finish, record; extra when enabled: dragAndDrop, rightClick. Use tool names exactly as listed. Do not invent combined names or aliases. Reloading is not a tool: to re-read a page from the server, instruct Tester to run I.refreshPage() through form. diff --git a/src/ai/planner.ts b/src/ai/planner.ts index ee374c27..f8a2f687 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -13,9 +13,11 @@ import { createDebug, tag } from '../utils/logger.js'; import { loop } from '../utils/loop.ts'; import { jsonToTable } from '../utils/markdown-parser.ts'; import { mdq } from '../utils/markdown-query.js'; +import { RulesLoader } from '../utils/rules-loader.ts'; import { findPlanMarkdownProblems, parsePlansFromMarkdownText, planToCompactAiContext, readPlanMarkdownComment } from '../utils/test-plan-markdown.ts'; import type { Agent, AgentDeps } from './agent.js'; import { Conversation } from './conversation.ts'; +import { EXPLICIT_TOOL_NAMES } from './explicit-tools.ts'; import type { Fisherman } from './fisherman.ts'; import { WithSessionDedup } from './planner/session-dedup.ts'; import { getActiveStyle } from './planner/styles.ts'; @@ -24,9 +26,8 @@ import type { Provider } from './provider.js'; import { POSSIBLE_SECTIONS, type Researcher } from './researcher.ts'; import { findSimilarStateHash } from './researcher/cache.ts'; import { hasFocusedSection } from './researcher/focus.ts'; -import { RulesLoader } from '../utils/rules-loader.ts'; import { capabilityGroundingRule, dataProtectionRules, fileUploadRule } from './rules.ts'; -import { ScoutQuery, type Scout } from './scout.ts'; +import { type Scout, ScoutQuery } from './scout.ts'; const debugLog = createDebug('explorbot:planner'); @@ -244,6 +245,10 @@ export class Planner extends PlannerBase implements Agent { const tests = result.tests; debugLog('Created tests:', tests); + for (const test of tests) { + if (await this.needsExplicitTools(test)) test.explicitTools = [...EXPLICIT_TOOL_NAMES]; + } + if (!this.currentPlan) { const cached = state.url ? getRegisteredPlan(state.url) : null; const planName = feature || cached?.plan.title || result.planName || state.url; @@ -282,6 +287,15 @@ export class Planner extends PlannerBase implements Agent { return this.lastSuite; } + private async needsExplicitTools(test: Test): Promise { + const needs = await this.provider.oracle( + 'Performing the steps of this test scenario or verifying its expected outcomes requires a non-standard interaction: dragging an element to a new position, or opening a context menu with a right-click.', + { scenario: test.scenario, steps: test.plannedSteps, expectedOutcomes: test.expected }, + { agentName: 'planner', telemetry: { functionId: 'planner.explicitTools' } } + ); + return needs === true; + } + private addNewTests(tests: Test[], defaultStartUrl: string): Test[] { if (!this.currentPlan) return []; diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index bd21a92a..5bfe6a04 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -440,7 +440,7 @@ export class Rerunner extends TaskAgent implements Agent { - You MUST execute the replacement action — not just diagnose - Use click() for buttons, links — commands array is FALLBACK LOCATORS for the SAME element - - Use form() for text input, dropdown selection, file uploads + - Use form() for text input, dropdown selection, file uploads; drag steps are recorded as plain I.usePlaywrightTo mouse code - Use pressKey() for special keys or key combinations - Use wait() when page is loading — returns fresh ARIA automatically - Use research() to understand page structure, sections, and available UI elements diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index 03c1d28b..55d1737f 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -585,6 +585,44 @@ export class Researcher extends ResearcherBase implements Agent { return r.text; } + async checkDragPoints(state: WebPageState, fromDescription: string, toDescription: string): Promise { + const screenshotData = this.getScreenshotFromState(state); + if (!screenshotData) return null; + + const { actionResult, image } = screenshotData; + tag('step').log('Locating drag points on screenshot'); + const prompt = dedent` + + You are a precise UI inspector preparing a drag-and-drop operation on a webpage screenshot. + + + + Locate the element to grab: "${fromDescription}". + Locate the drop target: "${toDescription}". + When both are visible, respond with one short sentence naming each and give the point + coordinates: the grab point at the center of what is dragged, the drop point at the center + of where it must land. End the answer with "grab X, Y; drop X, Y" + (numbers must be integers followed by X and Y respectively). + If either element is not visible or cannot be reached, say which one and why — no coordinates then. + Do not describe other elements. + + + + - Keep the answer under three sentences. + - Give both points only when both elements are located. + - Coordinates must follow the pattern "123X, 456Y" with X and Y suffixes. + + + URL: ${actionResult.url || 'Unknown'} + Title: ${actionResult.title || 'Unknown'} + + The screenshot is provided below. + `; + + const r = await this.provider.processImage(prompt, image.toString('base64')); + return r.text; + } + async answerQuestionAboutScreenshot(state: WebPageState, question: string): Promise { const screenshotData = this.getScreenshotFromState(state); if (!screenshotData) return null; diff --git a/src/ai/researcher/fingerprint-worker.ts b/src/ai/researcher/fingerprint-worker.ts index 286dd5d3..12d080f1 100644 --- a/src/ai/researcher/fingerprint-worker.ts +++ b/src/ai/researcher/fingerprint-worker.ts @@ -2,8 +2,8 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import { join } from 'node:path'; import { parentPort } from 'node:worker_threads'; import { computeHtmlFingerprint } from '../../utils/html-diff.ts'; -import { isSamePageFamily } from '../../utils/url-matcher.ts'; import { similarityScore } from '../../utils/similarity.ts'; +import { isSamePageFamily } from '../../utils/url-matcher.ts'; parentPort!.on('message', (data: FingerprintRequest) => { const { id, html, statesDir, maxAgeMs, threshold, url } = data; diff --git a/src/ai/rules.ts b/src/ai/rules.ts index b996f977..7e412c17 100644 --- a/src/ai/rules.ts +++ b/src/ai/rules.ts @@ -504,6 +504,12 @@ export const actionRule = dedent` After hovering, use see() or context() to check what appeared. + ### I.dragAndDrop + + When form() must drag directly, I.dragAndDrop(, ) takes Playwright locators + as written: a bare string is read as CSS and an object locator loses its text filter. + Write text as 'text=...' and prefer CSS or XPath. + ### I.seeInClipboard / I.seeClipboardEquals / I.clearClipboard Check what a copy control put into the system clipboard. Run via form(), not click(). @@ -523,3 +529,34 @@ export const actionRule = dedent` `; + +export const explicitToolsRule = dedent` + + These tools are present only when the plan enables them or the Pilot requests them mid-run. + They cover interactions click() and form() cannot express. + + ### dragAndDrop() tool + + Drags an element and drops it onto another element. Run via the dragAndDrop() tool, not form(). + The tool takes visual descriptions of what to grab and where to drop it, locates both points + on the screenshot, and drags between them with a real mouse — this also covers sliders and canvas + drawing, where nothing can be addressed by a locator. When items look alike, describe the position + ("the second card in the first column"). When a drag starts from a handle, describe the handle + as the grab point, not the whole card. For a drag with a modifier held (a copy-drag that + duplicates the item), pass the modifier to the dragAndDrop() tool. + + + dragAndDrop(from: "card labeled Bug 5 in the first column", to: "column labeled Done") + + + ### rightClick() tool + + Opens a context menu on an element with a right-click. Run via the rightClick() tool, not form(). + All commands in one call must target the SAME element, ordered by locator reliability like click(). + After the menu opens, act on its items with click() — the menu items are not part of this tool. + + + rightClick(commands: ['I.rightClick("Bug 5", ".board")']) + + + `; diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 11780b1f..4923d7da 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -18,11 +18,12 @@ import { compactErrorMessage } from '../utils/strings.ts'; import { isSamePageIgnoringQuery } from '../utils/url-matcher.ts'; import type { Agent, AgentDeps } from './agent.ts'; import type { Conversation } from './conversation.ts'; +import { createExplicitTools, normalizeExplicitTools, parseExplicitToolsRequest } from './explicit-tools.ts'; import { pageEvidence } from './judge.ts'; import { Navigator } from './navigator.ts'; import type { Pilot } from './pilot.ts'; import { Researcher } from './researcher.ts'; -import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts'; +import { actionRule, capabilityGroundingRule, dataProtectionRules, explicitToolsRule, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from './rules.ts'; import { TaskAgent } from './task-agent.ts'; import { createCodeceptJSTools, createIframeTools, withdrawVisionTools } from './tools.ts'; @@ -64,6 +65,7 @@ export class Tester extends TaskAgent implements Agent { private seenUiMapUrls = new Set(); private lastAnalyzedStateHash: string | null = null; private stalledIterations = 0; + private enabledExplicitTools = new Set(); private previousRegionPresent: boolean | null = null; private regionTransitioned = false; private readonly MAX_STALLED_ITERATIONS = 3; @@ -120,6 +122,7 @@ export class Tester extends TaskAgent implements Agent { this.seenUiMapUrls.clear(); this.lastAnalyzedStateHash = null; this.stalledIterations = 0; + this.enabledExplicitTools = new Set(normalizeExplicitTools(task.explicitTools)); this.pendingReview = ''; this.previousRegionPresent = null; this.regionTransitioned = false; @@ -244,7 +247,10 @@ export class Tester extends TaskAgent implements Agent { const pageContext = await this.reinjectContextIfNeeded(1, testerState); if (pageContext) conversation.addUserText(pageContext); - if (plan) conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.`); + if (plan) { + const enabledRules = this.applyExplicitTools(plan); + conversation.addUserText(`Pilot's test plan:\n${plan}\n\nFollow this plan while executing the test.${enabledRules}`); + } const offStateChange = this.stateManager.onStateChange((event: StateTransition) => { if (task.hasFinished) return; @@ -254,6 +260,7 @@ export class Tester extends TaskAgent implements Agent { }); const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task); + const explicitTools = createExplicitTools({ ...this.toolDeps, researcher: this.researcher }, task); let assertionPerformed = false; let extensions = 0; let deadlineReached = false; @@ -282,6 +289,10 @@ export class Tester extends TaskAgent implements Agent { Object.assign(tools, createIframeTools(this.toolDeps)); } withdrawVisionTools(tools); + for (const name of this.enabledExplicitTools) { + if (Stats.visionDisabled && name === 'dragAndDrop') continue; + if (explicitTools[name]) tools[name] = explicitTools[name]; + } debugLog(`Test ${task.scenario} iteration ${iteration}`); @@ -320,7 +331,7 @@ export class Tester extends TaskAgent implements Agent { if (isNewPage && this.pilot) { const guidance = await this.pilot.reviewNewPage(task, currentState, conversation); - if (guidance) nextStep += `\n\n${guidance}`; + if (guidance) nextStep += `\n\n${guidance}${this.applyExplicitTools(guidance)}`; } else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) { let guidance: string; if (this.isStruggling) { @@ -328,7 +339,7 @@ export class Tester extends TaskAgent implements Agent { } else { guidance = await this.pilot.periodicAnalyzeProgress(task, currentState, conversation); } - if (guidance) nextStep += `\n\n${guidance}`; + if (guidance) nextStep += `\n\n${guidance}${this.applyExplicitTools(guidance)}`; this.lastAnalyzedStateHash = currentState.hash; } conversation.addUserText(nextStep); @@ -350,7 +361,7 @@ export class Tester extends TaskAgent implements Agent { const allToolNames = result?.toolExecutions?.map((execution: any) => execution.toolName) || []; const successfulToolNames = result?.toolExecutions?.filter((execution: any) => execution.wasSuccessful)?.map((execution: any) => execution.toolName) || []; - const actionPerformed = !!allToolNames.find((toolName: string) => this.ACTION_TOOLS.includes(toolName)); + const actionPerformed = !!allToolNames.find((toolName: string) => this.currentActionTools.includes(toolName)); assertionPerformed = !!successfulToolNames.find((toolName: string) => this.ASSERTION_TOOLS.includes(toolName)); const wasSuccessful = result?.toolExecutions?.every((execution: any) => execution.wasSuccessful); @@ -492,10 +503,21 @@ export class Tester extends TaskAgent implements Agent { return this.stalledIterations >= this.MAX_STALLED_ITERATIONS - 1; } + private get currentActionTools(): string[] { + return [...this.ACTION_TOOLS, ...this.enabledExplicitTools]; + } + + private applyExplicitTools(guidance: string): string { + const requested = parseExplicitToolsRequest(guidance).filter((name) => !this.enabledExplicitTools.has(name)); + if (requested.length === 0) return ''; + for (const name of requested) this.enabledExplicitTools.add(name); + return `\n\n${explicitToolsRule}`; + } + private shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean { const currentState = this.getCurrentState(); const stateChanged = previousState.url !== currentState.url || previousState.hash !== currentState.hash; - const actionTools = [...this.ACTION_TOOLS, ...this.DELEGATED_ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS]; + const actionTools = [...this.currentActionTools, ...this.DELEGATED_ACTION_TOOLS, ...this.SPECIAL_CONTEXT_ACTION_TOOLS]; const hasSuccessfulAction = toolExecutions.some((execution) => execution.wasSuccessful && actionTools.includes(execution.toolName)); const hasSuccessfulAssertion = toolExecutions.some((execution) => execution.wasSuccessful && this.ASSERTION_TOOLS.includes(execution.toolName)); @@ -514,7 +536,7 @@ export class Tester extends TaskAgent implements Agent { - Use tools ${this.ACTION_TOOLS.join(', ')} to interact with the page. + Use tools ${this.currentActionTools.join(', ')} to interact with the page. Fall back to interact() when those fail, when the step needs a sequence of actions, or when your context is not enough to locate the element. Use tool names exactly as listed in this prompt. Do not invent combined tool names or aliases. Match each tool input schema exactly. Do not invent parameter names or pass extra fields. @@ -743,6 +765,7 @@ export class Tester extends TaskAgent implements Agent { `; } + const enabledExplicitRule = this.enabledExplicitTools.size > 0 ? explicitToolsRule : ''; return dedent` @@ -837,6 +860,8 @@ export class Tester extends TaskAgent implements Agent { ${actionRule} + ${enabledExplicitRule} + ${sectionContextRule} ${formRequirementsRule} diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 9c9359ce..ced3620a 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -1211,7 +1211,7 @@ function cap(text: string | undefined | null, max: number): string { return `${text.slice(0, max)}\n[...truncated; ${text.length - max} chars omitted...]`; } -function throwIfFatalBrowserError(error: unknown): void { +export function throwIfFatalBrowserError(error: unknown): void { if (isFatalBrowserError(error)) throw error; } @@ -1240,7 +1240,7 @@ function transformContainsCommand(command: string): string { return command; } -function errorText(error: unknown): string { +export function errorText(error: unknown): string { if (error instanceof Error) return compactErrorMessage(error); return 'Unknown error occurred'; } @@ -1304,7 +1304,7 @@ export function formatExecutedSteps(steps: ExecutedStep[], requestedCount = step return lines.join('\n'); } -function hasObservablePageChange(data?: Record): boolean { +export function hasObservablePageChange(data?: Record): boolean { if (!data?.pageDiff) return false; if (data.pageDiff.urlChanged === true) return true; if (data.pageDiff.ariaChanges) return true; @@ -1361,6 +1361,7 @@ export function withdrawVisionTools(tools: Record): void { if (!Stats.visionDisabled) return; Reflect.deleteProperty(tools, 'see'); Reflect.deleteProperty(tools, 'visualClick'); + Reflect.deleteProperty(tools, 'dragAndDrop'); } export function clickFailureSuggestion(attempts: Array<{ error?: string }>): string { diff --git a/src/commands/plan-save-command.ts b/src/commands/plan-save-command.ts index 0cb759c5..9025050c 100644 --- a/src/commands/plan-save-command.ts +++ b/src/commands/plan-save-command.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import { printNextSteps } from '../utils/next-steps.ts'; -import { PlanCommand } from './plan-command.js'; import { BaseCommand, type Suggestion } from './base-command.js'; +import { PlanCommand } from './plan-command.js'; export class PlanSaveCommand extends BaseCommand { name = 'plan:save'; diff --git a/src/config.ts b/src/config.ts index bf983b7c..0f3b42da 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,8 +7,8 @@ import dedent from 'dedent'; import matter from 'gray-matter'; import { EXPLORBOT_CONFIG_PATHS, type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, loadSiteConfig, registerSite, resolveSiteTarget } from './global-config.js'; import { getCliName } from './utils/cli-name.js'; -import { deepMerge } from './utils/merge.js'; import { log, tag } from './utils/logger.js'; +import { deepMerge } from './utils/merge.js'; export const PROVIDERS: Record = { openai: { envKey: 'OPENAI_API_KEY', keyUrl: 'https://platform.openai.com/api-keys', load: async () => (await import('@ai-sdk/openai')).createOpenAI() }, diff --git a/src/playwright-recorder.ts b/src/playwright-recorder.ts index be412b8f..d2780b85 100644 --- a/src/playwright-recorder.ts +++ b/src/playwright-recorder.ts @@ -1,5 +1,5 @@ -import { createRequire } from 'node:module'; import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { createDebug } from './utils/logger.ts'; const debugLog = createDebug('explorbot:playwright-recorder'); diff --git a/src/test-plan.ts b/src/test-plan.ts index b0e5a2be..34580e14 100644 --- a/src/test-plan.ts +++ b/src/test-plan.ts @@ -259,6 +259,7 @@ export class Test extends Task { artifacts: Record; generatedCode?: string; style?: string; + explicitTools: string[] = []; planIteration = 0; enabled = true; startTime?: number; diff --git a/src/utils/step-analyzer.ts b/src/utils/step-analyzer.ts index 86b01c0e..be87cac4 100644 --- a/src/utils/step-analyzer.ts +++ b/src/utils/step-analyzer.ts @@ -2,12 +2,13 @@ import type { SessionStep } from '../experience-tracker.ts'; import type { StepData } from '../test-plan.ts'; import { isDynamicId } from './xpath.ts'; -export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const; +export const CODECEPT_TOOLS = ['click', 'hover', 'dragAndDrop', 'pressKey', 'form'] as const; export type CodeceptToolName = (typeof CODECEPT_TOOLS)[number]; const INTERNAL_STEP_PREFIXES = ['grab', 'save']; const CODECEPT_FORM_COMMANDS: readonly string[] = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption']; +const CODECEPT_DRAG_COMMANDS: readonly string[] = ['I.pressKeyDown', 'I.pressKeyUp']; export function isCodeceptToolName(toolName: string): toolName is CodeceptToolName { return CODECEPT_TOOLS.includes(toolName as CodeceptToolName); @@ -17,6 +18,7 @@ export function getCodeceptToolName(commandName: string): CodeceptToolName | nul const toolName = CODECEPT_TOOLS.find((name) => commandName === `I.${name}`); if (toolName) return toolName; if (CODECEPT_FORM_COMMANDS.includes(commandName)) return 'form'; + if (CODECEPT_DRAG_COMMANDS.includes(commandName)) return 'dragAndDrop'; return null; } diff --git a/src/utils/test-plan-markdown.ts b/src/utils/test-plan-markdown.ts index 5181bd05..c31445c1 100644 --- a/src/utils/test-plan-markdown.ts +++ b/src/utils/test-plan-markdown.ts @@ -132,6 +132,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { let inSteps = false; let inExpected = false; let priority: 'critical' | 'important' | 'high' | 'normal' | 'low' = 'normal'; + let explicitTools: string[] = []; for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); @@ -142,6 +143,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { inSteps = false; inExpected = false; priority = 'normal'; + explicitTools = []; const title = lines[i + 1]?.replace(/^#\s+/, '') || ''; currentPlan = new Plan(title); plans.push(currentPlan); @@ -161,6 +163,11 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { } const priorityMatch = block.match(/priority:\s*(\w+)/); priority = (priorityMatch?.[1] as 'critical' | 'important' | 'high' | 'normal' | 'low') || 'normal'; + const toolsMatch = block.match(/tools:\s*(.+)/); + explicitTools = (toolsMatch?.[1] || '') + .split(',') + .map((name) => name.trim().toLowerCase()) + .filter(Boolean); i = j; continue; } @@ -168,6 +175,7 @@ export function parsePlansFromMarkdownText(content: string): Plan[] { if (line.startsWith('# ') && currentTest === null) { const scenario = line.replace(/^#\s+/, ''); currentTest = new Test(scenario, priority, [], ''); + currentTest.explicitTools = explicitTools; currentPlan.addTest(currentTest); inRequirements = false; inSteps = false; @@ -274,7 +282,9 @@ function formatPlanSuite(plan: Plan): string { } for (const test of plan.tests) { - content += `\n`; + let testHeader = `\n`; content += `# ${test.scenario}\n\n`; content += '## Requirements\n'; content += `${test.startUrl || CURRENT_PAGE}\n\n`; diff --git a/test-data/drag-board.html b/test-data/drag-board.html new file mode 100644 index 00000000..c23e49b9 --- /dev/null +++ b/test-data/drag-board.html @@ -0,0 +1,67 @@ + + + + + Drag board + + +

Drag board

+ +

Reorderable list

+
    +
  • Item 1
    • +
  • Item 2
    • +
  • Item 3
    • +
+ +

HTML5 drop zone

+
Chip
+
Drop here
+ +

Guarded HTML5 drop zone

+
Guarded chip
+
Guarded drop here
+ + + + diff --git a/tests/integration/drag-and-drop-browser.test.ts b/tests/integration/drag-and-drop-browser.test.ts new file mode 100644 index 00000000..3f66bd50 --- /dev/null +++ b/tests/integration/drag-and-drop-browser.test.ts @@ -0,0 +1,88 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { join } from 'node:path'; +import { type Browser, type Page, chromium } from 'playwright'; +import { type DragPoints, syntheticDragDrop } from '../../src/ai/explicit-tools.ts'; + +let browser: Browser; + +beforeAll(async () => { + browser = await chromium.launch(); +}); + +afterAll(async () => { + await browser?.close(); +}); + +async function boardPage(): Promise { + const page = await browser.newPage(); + await page.goto(`file://${join(process.cwd(), 'test-data', 'drag-board.html')}`, { waitUntil: 'domcontentloaded' }); + return page; +} + +async function centerOf(page: Page, selector: string): Promise<{ x: number; y: number }> { + const box = await page.locator(selector).boundingBox(); + if (!box) throw new Error(`No bounding box for ${selector}`); + return { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) }; +} + +async function dragByPoints(page: Page, points: DragPoints, modifier?: string) { + if (modifier) await page.keyboard.down(modifier); + await page.mouse.move(points.from.x, points.from.y); + await page.mouse.down(); + await page.waitForTimeout(200); + await page.mouse.move(points.to.x, points.to.y, { steps: 10 }); + await page.waitForTimeout(200); + await page.mouse.up(); + if (modifier) await page.keyboard.up(modifier); +} + +async function listTexts(page: Page): Promise { + return page.locator('#mouse-list li').allTextContents(); +} + +describe('mouse-event drag and drop', () => { + it('reorders the list through a coordinate mouse drag', async () => { + const page = await boardPage(); + + await dragByPoints(page, { from: await centerOf(page, '#mouse-list li:nth-child(2)'), to: await centerOf(page, '#mouse-list li:nth-child(1)') }); + + expect(await listTexts(page)).toEqual(['Item 2', 'Item 1', 'Item 3']); + await page.close(); + }); + + it('duplicates the item when Control is held during the drag', async () => { + const page = await boardPage(); + + await dragByPoints(page, { from: await centerOf(page, '#mouse-list li:nth-child(2)'), to: await centerOf(page, '#mouse-list li:nth-child(1)') }, 'Control'); + + expect(await listTexts(page)).toEqual(['Item 2 (copy)', 'Item 1', 'Item 2', 'Item 3']); + await page.close(); + }); +}); + +describe('HTML5 drag and drop', () => { + it('responds to the native drag', async () => { + const page = await boardPage(); + + await page.dragAndDrop('#html5-chip', '#drop-zone'); + + expect(await page.locator('#drop-zone #html5-chip').count()).toBe(1); + await page.close(); + }); + + it('needs synthetic drag events when the source cancels the mouse press', async () => { + const page = await boardPage(); + + await dragByPoints(page, { from: await centerOf(page, '#guarded-chip'), to: await centerOf(page, '#guarded-zone') }); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); + + await page.dragAndDrop('#guarded-chip', '#guarded-zone'); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(0); + + const explorer: any = { withPage: (fn: (page: Page) => Promise) => fn(page) }; + const points = { from: await centerOf(page, '#guarded-chip'), to: await centerOf(page, '#guarded-zone') }; + expect(await syntheticDragDrop(explorer, points)).toBe(true); + expect(await page.locator('#guarded-zone #guarded-chip').count()).toBe(1); + await page.close(); + }); +}); diff --git a/tests/integration/pilot-ask-api.test.ts b/tests/integration/pilot-ask-api.test.ts index d13e1169..20180d1f 100644 --- a/tests/integration/pilot-ask-api.test.ts +++ b/tests/integration/pilot-ask-api.test.ts @@ -1,6 +1,6 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; import { createOpenAI } from '@ai-sdk/openai'; import { LLMock } from '@copilotkit/aimock'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; import { ActionResult } from '../../src/action-result.ts'; import { Pilot } from '../../src/ai/pilot.ts'; import { Provider } from '../../src/ai/provider.ts'; diff --git a/tests/integration/pilot-expectations.test.ts b/tests/integration/pilot-expectations.test.ts index 2d47c0e0..9d2d5b9c 100644 --- a/tests/integration/pilot-expectations.test.ts +++ b/tests/integration/pilot-expectations.test.ts @@ -1,6 +1,6 @@ -import { LLMock } from '@copilotkit/aimock'; -import { createOpenAI } from '@ai-sdk/openai'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'bun:test'; +import { createOpenAI } from '@ai-sdk/openai'; +import { LLMock } from '@copilotkit/aimock'; import { Pilot } from '../../src/ai/pilot.ts'; import { Provider } from '../../src/ai/provider.ts'; import { ConfigParser } from '../../src/config.ts'; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index 6e5017d2..cd969dd9 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -152,6 +152,7 @@ describe('Planner with aimock', () => { clearPlanRegistry(); clearSessionDedup(); clearStyleCache(); + provider.setJudge({ assert: async () => false } as any); planner = new Planner({ ...createMockDeps(), ai: provider } as any, { research: async () => taskBoardUiMap } as any); @@ -563,4 +564,20 @@ describe('Planner with aimock', () => { expect(prompt).toContain('page or subpage has not been observed'); expect(prompt).toContain('describe the expected visible result generically'); }); + + it('arms explicit tools when the decision model says the scenario needs them', async () => { + provider.setJudge({ assert: async () => true } as any); + + const plan = await planner.plan(); + + expect(plan.tests[0].explicitTools).toEqual(['dragAndDrop', 'rightClick']); + }); + + it('keeps explicit tools off when the decision model declines', async () => { + provider.setJudge({ assert: async () => false } as any); + + const plan = await planner.plan(); + + expect(plan.tests.every((t) => t.explicitTools.length === 0)).toBe(true); + }); }); diff --git a/tests/regression/lib/artifacts.ts b/tests/regression/lib/artifacts.ts index d19f197f..71559158 100644 --- a/tests/regression/lib/artifacts.ts +++ b/tests/regression/lib/artifacts.ts @@ -1,9 +1,9 @@ import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs'; import { join } from 'node:path'; import stripAnsi from 'strip-ansi'; -import { parsePlansFromMarkdown } from '../../../src/utils/test-plan-markdown.ts'; -import { mdq } from '../../../src/utils/markdown-query.ts'; import type { Plan } from '../../../src/test-plan.ts'; +import { mdq } from '../../../src/utils/markdown-query.ts'; +import { parsePlansFromMarkdown } from '../../../src/utils/test-plan-markdown.ts'; export function findResearchFiles(runDir: string): string[] { return listMarkdown(join(runDir, 'output', 'research')); diff --git a/tests/unit/captain-blocked.test.ts b/tests/unit/captain-blocked.test.ts index f8f3b3c1..3941bf94 100644 --- a/tests/unit/captain-blocked.test.ts +++ b/tests/unit/captain-blocked.test.ts @@ -2,9 +2,9 @@ import { describe, expect, it } from 'bun:test'; import { ActionResult } from '../../src/action-result.ts'; import { Captain } from '../../src/ai/captain.ts'; import type { ExperienceTracker } from '../../src/experience-tracker.ts'; +import { ExplorBot } from '../../src/explorbot.ts'; import type { KnowledgeTracker } from '../../src/knowledge-tracker.ts'; import { StateManager } from '../../src/state-manager.ts'; -import { ExplorBot } from '../../src/explorbot.ts'; import { Plan, Test } from '../../src/test-plan.ts'; import { NavigationBlockedError, PlanningBlockedError } from '../../src/utils/error-page.ts'; diff --git a/tests/unit/command-handler-autocomplete.test.ts b/tests/unit/command-handler-autocomplete.test.ts index 29e353e8..5871247e 100644 --- a/tests/unit/command-handler-autocomplete.test.ts +++ b/tests/unit/command-handler-autocomplete.test.ts @@ -1,7 +1,7 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import { mkdtempSync, rmSync, utimesSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import { CommandHandler } from '../../src/command-handler.js'; import type { ExplorBot } from '../../src/explorbot.js'; import { Plan, Test } from '../../src/test-plan.js'; diff --git a/tests/unit/doc-collector.test.ts b/tests/unit/doc-collector.test.ts index 821fe083..ff05b522 100644 --- a/tests/unit/doc-collector.test.ts +++ b/tests/unit/doc-collector.test.ts @@ -2,17 +2,17 @@ import { describe, expect, it } from 'bun:test'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { z } from 'zod'; import { PNG } from 'pngjs'; +import { z } from 'zod'; import { Documentarian } from '../../boat/doc-collector/src/ai/documentarian.ts'; import { pickDocActionCandidates } from '../../boat/doc-collector/src/ai/tools.ts'; import { DocBot } from '../../boat/doc-collector/src/docbot.ts'; import { normalizeAction, renderPageDocumentation, renderSpecIndex } from '../../boat/doc-collector/src/docs-renderer.ts'; import { getDocPageKey, shouldCrawlDocPath } from '../../boat/doc-collector/src/path-filter.ts'; import { extractResearchNavigationTargets } from '../../boat/doc-collector/src/research-navigation.ts'; -import { buildTemplateRecord, findTemplateMatch } from '../../boat/doc-collector/src/template-dedup.ts'; import { captureDocumentationScreenshots, captureEvidenceScreenshots, getScreenshotSections } from '../../boat/doc-collector/src/screenshots.ts'; import { renderMermaidBody } from '../../boat/doc-collector/src/state-diagram.ts'; +import { buildTemplateRecord, findTemplateMatch } from '../../boat/doc-collector/src/template-dedup.ts'; describe('doc-collector path filter', () => { it('allows regular documentation pages', () => { diff --git a/tests/unit/drag-and-drop.test.ts b/tests/unit/drag-and-drop.test.ts new file mode 100644 index 00000000..160c1d86 --- /dev/null +++ b/tests/unit/drag-and-drop.test.ts @@ -0,0 +1,177 @@ +import { beforeEach, describe, expect, it } from 'bun:test'; +import { createExplicitTools, dragPoints } from '../../src/ai/explicit-tools.ts'; +import { createCodeceptJSTools } from '../../src/ai/tools.ts'; +import { ConfigParser } from '../../src/config.ts'; + +const ANALYSIS = 'The first card and the Done column are visible. grab 10X, 20Y; drop 30X, 40Y'; + +function fakeDeps() { + const holder: { state: any } = { + state: { url: '/board', html: '', ariaSnapshot: '- listitem "First"', id: 'before' }, + }; + const action: any = { + lastError: null, + executedSteps: [] as Array<{ command: string; success: boolean }>, + ran: [] as string[], + saveScreenshot: async () => undefined, + attempt: async (command: string) => { + action.ran.push(command); + return false; + }, + }; + const researcher: any = { + checkDragPoints: async () => ANALYSIS, + }; + const deps: any = { + explorer: { action: () => action, capture: async () => ({ screenshot: Buffer.from('png') }) }, + stateManager: { getCurrentState: () => holder.state }, + ai: {}, + researcher, + }; + return { deps, action, holder, researcher }; +} + +function fakeTask(): any { + return { startNote: () => ({ commit: () => {}, screenshot: undefined }) }; +} + +function succeedWithReorder(action: any, holder: { state: any }) { + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + holder.state = { ...holder.state, ariaSnapshot: '- listitem "Second"\n- listitem "First"', id: 'after' }; + return true; + }; +} + +describe('dragPoints parsing', () => { + it('reads both points from the vision analysis', () => { + expect(dragPoints('grab 10X, 20Y; drop 30X, 40Y')).toEqual({ + from: { x: 10, y: 20 }, + to: { x: 30, y: 40 }, + }); + }); + + it('keeps dragAndDrop out of the default CodeceptJS tools', () => { + const tools = createCodeceptJSTools(fakeDeps().deps as any, fakeTask()); + expect(tools.dragAndDrop).toBeUndefined(); + }); + + it('reads the points from a sentence', () => { + expect(dragPoints('The card is in the first column. grab 5X, 6Y; drop 7X, 8Y')).toEqual({ + from: { x: 5, y: 6 }, + to: { x: 7, y: 8 }, + }); + }); + + it('returns null when a point is missing', () => { + expect(dragPoints('The drop target was not found')).toBeNull(); + expect(dragPoints('grab 10X, 20Y')).toBeNull(); + }); +}); + +describe('dragAndDrop tool', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + it('drags between the located points with a real mouse and records replayable code', async () => { + const { deps, action, holder } = fakeDeps(); + succeedWithReorder(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); + + expect(result.success).toBe(true); + expect(action.ran).toHaveLength(1); + expect(action.ran[0]).toContain('I.usePlaywrightTo'); + expect(action.ran[0]).toContain('page.mouse.move(10, 20)'); + expect(action.ran[0]).toContain('page.mouse.move(30, 40, { steps: 10 })'); + expect(result.code).toBe(action.ran[0]); + expect(result.points).toEqual({ from: { x: 10, y: 20 }, to: { x: 30, y: 40 } }); + }); + + it('wraps the drag between pressKeyDown and pressKeyUp for a modifier', async () => { + const { deps, action, holder } = fakeDeps(); + succeedWithReorder(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); + + expect(result.success).toBe(true); + expect(result.code).toBe(action.ran[0]); + expect(result.code.startsWith('I.pressKeyDown("Control")\n')).toBe(true); + expect(result.code.endsWith('I.pressKeyUp("Control")')).toBe(true); + }); + + it('fails without browser action when vision cannot locate both points', async () => { + const { deps, action } = fakeDeps(); + deps.researcher.checkDragPoints = async () => 'The drop target was not found'; + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('Drag points not found'); + expect(action.ran).toEqual([]); + }); + + it('escalates a silent mouse drag to HTML5 drag events', async () => { + const { deps, action, holder } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = command.split('\n').map((line) => ({ command: line, success: true })); + return true; + }; + let syntheticDone = false; + deps.explorer = { + action: () => action, + capture: async () => { + if (syntheticDone) holder.state = { ...holder.state, ariaSnapshot: '- moved', id: 'after' }; + return { screenshot: Buffer.from('png') }; + }, + withPage: async () => { + syntheticDone = true; + return true; + }, + }; + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); + + expect(result.success).toBe(true); + expect(result.message).toContain('HTML5 drag events'); + expect(result.attempts.map((attempt: any) => attempt.success)).toEqual([true, true]); + expect(result.attempts[1]?.command).toContain('HTML5 drag events'); + }); + + it('releases the modifier and fails when no mechanism changes the page', async () => { + const { deps, action } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + return command.startsWith('I.pressKeyUp'); + }; + deps.explorer.withPage = async () => false; + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', modifier: 'Control', explanation: 'copy the card' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('changed nothing'); + expect(action.ran[action.ran.length - 1]).toBe('I.pressKeyUp("Control")'); + }); + + it('reports drag as unavailable when there is no researcher', async () => { + const { deps, action } = fakeDeps(); + const tools = createExplicitTools({ ...deps, researcher: undefined }, fakeTask()); + + const result = await tools.dragAndDrop.execute({ from: 'first card', to: 'Done column', explanation: 'move the card' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('cannot be located'); + expect(action.ran).toEqual([]); + }); +}); diff --git a/tests/unit/executed-steps.test.ts b/tests/unit/executed-steps.test.ts index c39a237a..dccec1f7 100644 --- a/tests/unit/executed-steps.test.ts +++ b/tests/unit/executed-steps.test.ts @@ -1,6 +1,6 @@ -import codeceptjs from 'codeceptjs'; import { describe, expect, it } from 'bun:test'; -import { attachStepLogger, type ExecutedStep } from '../../src/action.ts'; +import codeceptjs from 'codeceptjs'; +import { type ExecutedStep, attachStepLogger } from '../../src/action.ts'; import { formatExecutedSteps } from '../../src/ai/tools.ts'; function step(code: string) { @@ -35,6 +35,18 @@ describe('executed steps', () => { expect(steps[0]?.command).toBe('I.click("Manual", step.opts({"elementIndex":2}))'); }); + it('escapes a locator containing quotes so the recorded command stays valid code', () => { + const steps: ExecutedStep[] = []; + const detachSteps = attachStepLogger(steps); + + const drag = { prefix: '', suffix: '', actor: 'I', name: 'dragAndDrop', args: ['li:has-text("Item 2")', 'li:has-text("Item 1")'], humanizeArgs: () => '"li:has-text("Item 2")"', toCode: () => 'I.dragAndDrop("li:has-text("Item 2")", "li:has-text("Item 1")")' }; + codeceptjs.event.dispatcher.emit(codeceptjs.event.step.passed, drag); + + detachSteps(); + + expect(steps[0]?.command).toBe('I.dragAndDrop("li:has-text(\\"Item 2\\")", "li:has-text(\\"Item 1\\")")'); + }); + it('reports a successful command even when a later one fails', () => { const report = formatExecutedSteps( [ diff --git a/tests/unit/explicit-tools.test.ts b/tests/unit/explicit-tools.test.ts new file mode 100644 index 00000000..7f1b7e10 --- /dev/null +++ b/tests/unit/explicit-tools.test.ts @@ -0,0 +1,200 @@ +import { beforeEach, describe, expect, it } from 'bun:test'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { EXPLICIT_TOOL_NAMES, createExplicitTools, parseExplicitToolsRequest } from '../../src/ai/explicit-tools.ts'; +import { Tester } from '../../src/ai/tester.ts'; +import { ConfigParser } from '../../src/config.ts'; +import { Plan, Test } from '../../src/test-plan.ts'; +import { parsePlansFromMarkdownText, savePlanToMarkdown } from '../../src/utils/test-plan-markdown.ts'; + +function fakeDeps() { + const holder: { state: any } = { + state: { url: '/board', html: '', ariaSnapshot: '- listitem "First"', id: 'before' }, + }; + const action: any = { + lastError: null, + executedSteps: [] as Array<{ command: string; success: boolean }>, + ran: [] as string[], + saveScreenshot: async () => undefined, + attempt: async (command: string) => { + action.ran.push(command); + return false; + }, + }; + const deps: any = { + explorer: { action: () => action, capture: async () => ({ screenshot: Buffer.from('png') }) }, + stateManager: { getCurrentState: () => holder.state }, + ai: {}, + }; + return { deps, action, holder }; +} + +function fakeTask(): any { + return { startNote: () => ({ commit: () => {}, screenshot: undefined }) }; +} + +function openMenu(action: any, holder: { state: any }) { + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = [{ command, success: true }]; + holder.state = { ...holder.state, ariaSnapshot: '- menu "Rename"\n- menu "Delete"', id: 'after' }; + return true; + }; +} + +describe('rightClick tool', () => { + beforeEach(() => { + ConfigParser.resetForTesting(); + ConfigParser.setupTestConfig(); + }); + + it('right-clicks the first working locator and reports the opened menu', async () => { + const { deps, action, holder } = fakeDeps(); + openMenu(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5", ".board")', 'I.rightClick(".card-5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(true); + expect(action.ran).toEqual(['I.rightClick("Bug 5", ".board")']); + expect(result.code).toBe('I.rightClick("Bug 5", ".board")'); + }); + + it('wraps a bare locator into I.rightClick()', async () => { + const { deps, action, holder } = fakeDeps(); + openMenu(action, holder); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['Bug 5'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(true); + expect(action.ran).toEqual(['I.rightClick("Bug 5")']); + }); + + it('rejects non-right-click commands without touching the browser', async () => { + const { deps, action } = fakeDeps(); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.click("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('only accepts I.rightClick()'); + expect(action.ran).toEqual([]); + }); + + it('fails when all locators fail', async () => { + const { deps, action } = fakeDeps(); + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('All right-click commands failed'); + expect(action.ran).toEqual(['I.rightClick("Bug 5")']); + }); + + it('fails when the click lands but no context menu appears', async () => { + const { deps, action } = fakeDeps(); + action.attempt = async (command: string) => { + action.ran.push(command); + action.lastError = null; + action.executedSteps = [{ command, success: true }]; + return true; + }; + const tools = createExplicitTools(deps, fakeTask()); + + const result = await tools.rightClick.execute({ commands: ['I.rightClick("Bug 5")'], explanation: 'open the card menu' }, {} as any); + + expect(result.success).toBe(false); + expect(result.message).toContain('no context menu appeared'); + }); +}); + +describe('parseExplicitToolsRequest', () => { + it('reads requested tool names from pilot guidance', () => { + expect(parseExplicitToolsRequest('Move the card.\nENABLE_TOOLS: dragAndDrop, rightClick\nFollow this plan.')).toEqual(['dragAndDrop', 'rightClick']); + }); + + it('ignores unknown names and is case-insensitive', () => { + expect(parseExplicitToolsRequest('ENABLE_TOOLS: draganddrop, hover, click')).toEqual(['dragAndDrop']); + }); + + it('returns nothing without the marker', () => { + expect(parseExplicitToolsRequest('Use click() on the card.')).toEqual([]); + }); +}); + +describe('explicit tools in the test plan', () => { + let tmpPath: string; + + beforeEach(() => { + tmpPath = mkdtempSync(join(tmpdir(), 'explorbot-explicit-')); + }); + + it('round-trips the tools flag through the plan markdown', () => { + const plan = new Plan('/board'); + const dragTest = new Test('Reorder cards by dragging', 'high', ['cards are reordered'], '/board', ['drag the card']); + dragTest.explicitTools = [...EXPLICIT_TOOL_NAMES]; + const plainTest = new Test('Open the board', 'normal', ['board is visible'], '/board', []); + plan.addTest(dragTest); + plan.addTest(plainTest); + const file = join(tmpPath, 'plan.md'); + savePlanToMarkdown(plan, file); + + const restored = parsePlansFromMarkdownText(readFileSync(file, 'utf-8'))[0]; + + expect(restored.tests[0].explicitTools).toEqual(['draganddrop', 'rightclick']); + expect(restored.tests[1].explicitTools).toEqual([]); + }); + + it('defaults to no explicit tools for plans written without the flag', () => { + const content = ['', '# /board', '', '', '# Reorder cards', '', '## Requirements', '', '/board', ''].join('\n'); + const restored = parsePlansFromMarkdownText(content)[0]; + + expect(restored.tests[0].explicitTools).toEqual([]); + }); +}); + +describe('tester explicit tools gating', () => { + function buildTester() { + const deps: any = { + explorer: {}, + ai: { getSystemPromptForAgent: () => '' }, + config: {}, + stateManager: { + getCurrentState: () => null, + otherTabs: [], + getExperienceTracker: () => ({ getExperienceTableOfContents: () => [] }), + }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', renderRelevantContext: () => '' }, + requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, + playwrightRecorder: {}, + }; + return new Tester(deps, {} as any, {} as any); + } + + it('keeps the explicit tools rule out of the system message by default', () => { + const tester = buildTester(); + + expect(tester.getSystemMessage()).not.toContain(''); + }); + + it('injects the rule when the plan armed the tools', () => { + const tester = buildTester(); + (tester as any).enabledExplicitTools = new Set(['dragAndDrop']); + + expect(tester.getSystemMessage()).toContain('### dragAndDrop() tool'); + }); + + it('enables tools from the pilot marker and returns the rules to inject', () => { + const tester = buildTester(); + const rules = (tester as any).applyExplicitTools('ENABLE_TOOLS: dragAndDrop, rightClick'); + + expect((tester as any).enabledExplicitTools).toEqual(new Set(['dragAndDrop', 'rightClick'])); + expect(rules).toContain(''); + expect((tester as any).applyExplicitTools('ENABLE_TOOLS: dragAndDrop')).toBe(''); + expect((tester as any).applyExplicitTools('No marker here')).toBe(''); + }); +}); diff --git a/tests/unit/judge-tool.test.ts b/tests/unit/judge-tool.test.ts index 2ce6f871..4465cd41 100644 --- a/tests/unit/judge-tool.test.ts +++ b/tests/unit/judge-tool.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'bun:test'; -import { Decision } from '../../src/ai/judge.ts'; import { createJudgeTool } from '../../src/ai/judge-tool.ts'; +import { Decision } from '../../src/ai/judge.ts'; function judgeReturning(decision: Decision, seen: any[] = []) { return { diff --git a/tests/unit/overlay-detection.test.ts b/tests/unit/overlay-detection.test.ts index 9749cd59..b39ef5ab 100644 --- a/tests/unit/overlay-detection.test.ts +++ b/tests/unit/overlay-detection.test.ts @@ -1,7 +1,7 @@ import 'parse5'; import { beforeEach, describe, expect, it } from 'vitest'; -import Action from '../../src/action.ts'; import { ActionResult } from '../../src/action-result.ts'; +import Action from '../../src/action.ts'; import { ConfigParser } from '../../src/config.ts'; import { htmlDiff } from '../../src/utils/html-diff.ts'; import { Overlay, OverlayPage } from '../../src/utils/overlay.ts'; diff --git a/tests/unit/plans-command.test.ts b/tests/unit/plans-command.test.ts index dd10eb19..8c19eed9 100644 --- a/tests/unit/plans-command.test.ts +++ b/tests/unit/plans-command.test.ts @@ -1,7 +1,7 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, mock } from 'bun:test'; import { PlansCommand } from '../../src/commands/plans-command.js'; import { TestCommand } from '../../src/commands/test-command.js'; import type { ExplorBot } from '../../src/explorbot.js'; diff --git a/tests/unit/remote.test.ts b/tests/unit/remote.test.ts index 78cfe968..30d880de 100644 --- a/tests/unit/remote.test.ts +++ b/tests/unit/remote.test.ts @@ -4,8 +4,8 @@ import { Command } from 'commander'; import Action from '../../src/action.ts'; import { saveResearch } from '../../src/ai/researcher/cache.ts'; import { SessionAnalyst } from '../../src/ai/session-analyst.ts'; -import { wsOption } from '../../src/commands/options/index.ts'; import { isInteractive } from '../../src/ai/task-agent.ts'; +import { wsOption } from '../../src/commands/options/index.ts'; import { ConfigParser } from '../../src/config.ts'; import { executionController } from '../../src/execution-controller.ts'; import { remote } from '../../src/remote.ts';