diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed049c7..e1d6fe2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2026-11-07 + +### Changes +- [Researcher] Opens one item from each list on the page, such as a product card, so controls that only appear inside it are found. It only opens links and controls marked as opening something, never unlabelled row buttons like Delete. +- [Planner] Plans tests for actions that live inside a list item. On a store whose Add to cart button sits in the product dialog, the plan now covers adding to the cart. + ## 2026-11-06 ### Changes diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 533a3eb3..85fa1491 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -1101,6 +1101,6 @@ function isInteractiveElement(element: WebElement): boolean { if (HTML_INTERACTIVE_ROLES.has(role)) return true; if (element.attrs.contenteditable === 'true') return true; if (element.attrs.tabindex && Number(element.attrs.tabindex) >= 0) return true; - if (element.attrs['aria-haspopup'] || element.attrs['aria-expanded'] || element.attrs['aria-controls']) return true; + if (element.revealsContent) return true; return false; } diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index 508b5bc9..f71111d8 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -10,10 +10,12 @@ import { extractCodeBlocks } from '../../utils/code-extractor.ts'; import { tag } from '../../utils/logger.js'; import { mdq } from '../../utils/markdown-query.ts'; import { truncate } from '../../utils/strings.ts'; +import { annotatePageElements } from '../../utils/web-annotate.ts'; +import { eidxInContainer } from '../../utils/web-eidx.ts'; import type { Provider } from '../provider.js'; import { getCachedResearch, getPreviousResearch, saveResearch } from './cache.ts'; import { type Constructor, debugLog } from './mixin.ts'; -import { type ResearchElement, parseResearchSections } from './parser.ts'; +import { type ResearchElement, parseDataSections, parseResearchSections } from './parser.ts'; import type { ResearchResult } from './research-result.ts'; const DEFAULT_MAX_EXPANDABLE_CLICKS = 10; @@ -60,6 +62,7 @@ export function WithDeepAnalysis(Base: T) { expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks); tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`); } + expandables.push(...(await this._findDataItems(result.text))); const elements = expandables .map((el) => ({ @@ -296,6 +299,28 @@ export function WithDeepAnalysis(Base: T) { return [...eidxSet].map((eidx) => allElements.get(eidx)!); } + private async _findDataItems(researchText: string): Promise { + const sections = parseDataSections(researchText).filter((section) => section.containerCss); + if (sections.length === 0) return []; + + const { elements } = await this.explorer.withPage(annotatePageElements); + await this.explorer.capture(); + const byEidx = new Map(elements.map((el) => [el.eidx, el])); + + const items: ExpandableElement[] = []; + for (const section of sections) { + const eidxList: string[] = await this.explorer.withPage((page) => eidxInContainer(page, section.containerCss)).catch(() => []); + const item = eidxList.map((eidx) => byEidx.get(eidx)).find((el) => el?.declaresDestination); + if (!item) continue; + + const label = item.attrs['aria-label'] || item.text; + let aria: ResearchElement['aria'] = null; + if (item.role && label) aria = { role: item.role, text: label }; + items.push({ name: label || section.name, type: item.role, aria, css: item.clickXPath, xpath: null, coordinates: null, color: null, icon: null, eidx: item.eidx, container: section.containerCss }); + } + return items; + } + private _buildClickCommands(el: ExpandableElement): string[] { const commands: string[] = []; const escapeQuote = (s: string) => s.replace(/'/g, "\\'"); diff --git a/src/utils/web-element.ts b/src/utils/web-element.ts index fc2ce338..b3e8268a 100644 --- a/src/utils/web-element.ts +++ b/src/utils/web-element.ts @@ -5,6 +5,7 @@ export { extractElementData } from './html.ts'; const KEY_DISPLAY_ATTRS = ['role', 'id', 'class', 'aria-label']; const KEY_ATTRS = ['role', 'aria-label', 'id', 'name', 'type', 'href']; +const REVEALING_ATTRS = ['aria-haspopup', 'aria-expanded', 'aria-controls']; export class WebElement { tag: string; @@ -57,6 +58,14 @@ export class WebElement { return !!href && href !== '#' && !href.startsWith('javascript:'); } + get revealsContent(): boolean { + return REVEALING_ATTRS.some((attr) => this.attrs[attr]); + } + + get declaresDestination(): boolean { + return this.isNavigationLink || this.revealsContent; + } + get filteredClasses(): string[] { const cls = this.attrs.class || ''; return cls.split(/\s+/).filter((c) => c.length > 2 && !isDynamicId(c) && !isGenericClass(c)); diff --git a/tests/integration/deep-analysis-data-items-browser.test.ts b/tests/integration/deep-analysis-data-items-browser.test.ts new file mode 100644 index 00000000..6a1cc2b8 --- /dev/null +++ b/tests/integration/deep-analysis-data-items-browser.test.ts @@ -0,0 +1,105 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { type Browser, type Page, chromium } from 'playwright'; +import { WithDeepAnalysis } from '../../src/ai/researcher/deep-analysis.ts'; + +const DeepAnalysis = WithDeepAnalysis(Object as any) as any; + +const STATE = { url: '/list', hash: '', html: '', ariaSnapshot: '' }; + +function dataSection(container: string): string { + return `## Data: Items\n\n> Container: \`${container}\`\n\nItems, 2 items.\n`; +} + +describe('deep analysis of Data sections', () => { + let browser: Browser; + let page: Page; + + beforeAll(async () => { + browser = await chromium.launch(); + page = await browser.newPage(); + }); + + afterAll(async () => { + await browser.close(); + }); + + async function attemptedClicks(html: string, research: string): Promise { + await page.setContent(html); + const attempted: string[] = []; + const analysis = new DeepAnalysis(); + analysis.config = { ai: {} }; + analysis.provider = { getModelForAgent: () => null, chat: async () => ({ text: '' }), hasVision: () => false }; + analysis.stateManager = { getCurrentState: () => STATE }; + analysis.navigateTo = async () => {}; + analysis.cancelInUi = async () => {}; + analysis.explorer = { + withPage: async (fn: any) => fn(page), + capture: async () => {}, + action: () => ({ + attempt: async (code: string) => { + if (code.startsWith('I.click')) attempted.push(code); + return false; + }, + }), + }; + await analysis.performDeepAnalysis(STATE, { text: research }); + return attempted; + } + + it('opens one item when the item declares a popup', async () => { + const clicks = await attemptedClicks( + `
+
First
+
Second
+
`, + dataSection('#grid') + ); + + expect(clicks.length).toBeGreaterThan(0); + expect(clicks.some((code) => code.includes('View first item'))).toBe(true); + expect(clicks.some((code) => code.includes('View second item'))).toBe(false); + }); + + it('follows the first link of a list', async () => { + const clicks = await attemptedClicks( + ``, + dataSection('#list') + ); + + expect(clicks.some((code) => code.includes('First record'))).toBe(true); + expect(clicks.some((code) => code.includes('Second record'))).toBe(false); + }); + + it('never presses an item control that does not declare where it leads', async () => { + const clicks = await attemptedClicks( + ` + + +
First
Second
`, + dataSection('#rows') + ); + + expect(clicks).toEqual([]); + }); + + it('skips a safe control when it sits after an unsafe one', async () => { + const clicks = await attemptedClicks( + ``, + dataSection('#list') + ); + + expect(clicks.some((code) => code.includes('Archive'))).toBe(false); + expect(clicks.some((code) => code.includes('First record'))).toBe(true); + }); + + it('ignores a Data section whose container is not on the page', async () => { + const clicks = await attemptedClicks('Elsewhere', dataSection('#gone')); + + expect(clicks).toEqual([]); + }); +});