From b078a39e872b649fdce411082ac8292469c8f8bf Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 7 Nov 2026 13:46:42 +0200 Subject: [PATCH] Open one list item during deep research to find controls inside it Deep analysis never considered Data sections: they have no UI map table and parseResearchSections filters them out. Anything reachable only from a list item stayed invisible, so the Planner could not ground it. On a store with Add to cart in the product dialog, no cart tests were planned. Deep analysis now clicks the first item control of each Data section. It takes only real links and controls with aria-haspopup, aria-expanded or aria-controls, so an unlabelled row button such as Delete is never pressed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + src/ai/driller.ts | 2 +- src/ai/researcher/deep-analysis.ts | 27 ++++- src/utils/web-element.ts | 9 ++ .../deep-analysis-data-items-browser.test.ts | 105 ++++++++++++++++++ 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 tests/integration/deep-analysis-data-items-browser.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed049c7..e1d6fe2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2026-11-07 + +### Changes +- [Researcher] Opens one item from each list on the page, such as a product card, so controls that only appear inside it are found. It only opens links and controls marked as opening something, never unlabelled row buttons like Delete. +- [Planner] Plans tests for actions that live inside a list item. On a store whose Add to cart button sits in the product dialog, the plan now covers adding to the cart. + ## 2026-11-06 ### Changes diff --git a/src/ai/driller.ts b/src/ai/driller.ts index 533a3eb3..85fa1491 100644 --- a/src/ai/driller.ts +++ b/src/ai/driller.ts @@ -1101,6 +1101,6 @@ function isInteractiveElement(element: WebElement): boolean { if (HTML_INTERACTIVE_ROLES.has(role)) return true; if (element.attrs.contenteditable === 'true') return true; if (element.attrs.tabindex && Number(element.attrs.tabindex) >= 0) return true; - if (element.attrs['aria-haspopup'] || element.attrs['aria-expanded'] || element.attrs['aria-controls']) return true; + if (element.revealsContent) return true; return false; } diff --git a/src/ai/researcher/deep-analysis.ts b/src/ai/researcher/deep-analysis.ts index 508b5bc9..f71111d8 100644 --- a/src/ai/researcher/deep-analysis.ts +++ b/src/ai/researcher/deep-analysis.ts @@ -10,10 +10,12 @@ import { extractCodeBlocks } from '../../utils/code-extractor.ts'; import { tag } from '../../utils/logger.js'; import { mdq } from '../../utils/markdown-query.ts'; import { truncate } from '../../utils/strings.ts'; +import { annotatePageElements } from '../../utils/web-annotate.ts'; +import { eidxInContainer } from '../../utils/web-eidx.ts'; import type { Provider } from '../provider.js'; import { getCachedResearch, getPreviousResearch, saveResearch } from './cache.ts'; import { type Constructor, debugLog } from './mixin.ts'; -import { type ResearchElement, parseResearchSections } from './parser.ts'; +import { type ResearchElement, parseDataSections, parseResearchSections } from './parser.ts'; import type { ResearchResult } from './research-result.ts'; const DEFAULT_MAX_EXPANDABLE_CLICKS = 10; @@ -60,6 +62,7 @@ export function WithDeepAnalysis(Base: T) { expandables = await this._selectExpandables(expandables, state.fullUrl || state.url, maxClicks); tag('substep').log(`Selected ${expandables.length} expandables to click (max: ${maxClicks})`); } + expandables.push(...(await this._findDataItems(result.text))); const elements = expandables .map((el) => ({ @@ -296,6 +299,28 @@ export function WithDeepAnalysis(Base: T) { return [...eidxSet].map((eidx) => allElements.get(eidx)!); } + private async _findDataItems(researchText: string): Promise { + const sections = parseDataSections(researchText).filter((section) => section.containerCss); + if (sections.length === 0) return []; + + const { elements } = await this.explorer.withPage(annotatePageElements); + await this.explorer.capture(); + const byEidx = new Map(elements.map((el) => [el.eidx, el])); + + const items: ExpandableElement[] = []; + for (const section of sections) { + const eidxList: string[] = await this.explorer.withPage((page) => eidxInContainer(page, section.containerCss)).catch(() => []); + const item = eidxList.map((eidx) => byEidx.get(eidx)).find((el) => el?.declaresDestination); + if (!item) continue; + + const label = item.attrs['aria-label'] || item.text; + let aria: ResearchElement['aria'] = null; + if (item.role && label) aria = { role: item.role, text: label }; + items.push({ name: label || section.name, type: item.role, aria, css: item.clickXPath, xpath: null, coordinates: null, color: null, icon: null, eidx: item.eidx, container: section.containerCss }); + } + return items; + } + private _buildClickCommands(el: ExpandableElement): string[] { const commands: string[] = []; const escapeQuote = (s: string) => s.replace(/'/g, "\\'"); diff --git a/src/utils/web-element.ts b/src/utils/web-element.ts index fc2ce338..b3e8268a 100644 --- a/src/utils/web-element.ts +++ b/src/utils/web-element.ts @@ -5,6 +5,7 @@ export { extractElementData } from './html.ts'; const KEY_DISPLAY_ATTRS = ['role', 'id', 'class', 'aria-label']; const KEY_ATTRS = ['role', 'aria-label', 'id', 'name', 'type', 'href']; +const REVEALING_ATTRS = ['aria-haspopup', 'aria-expanded', 'aria-controls']; export class WebElement { tag: string; @@ -57,6 +58,14 @@ export class WebElement { return !!href && href !== '#' && !href.startsWith('javascript:'); } + get revealsContent(): boolean { + return REVEALING_ATTRS.some((attr) => this.attrs[attr]); + } + + get declaresDestination(): boolean { + return this.isNavigationLink || this.revealsContent; + } + get filteredClasses(): string[] { const cls = this.attrs.class || ''; return cls.split(/\s+/).filter((c) => c.length > 2 && !isDynamicId(c) && !isGenericClass(c)); diff --git a/tests/integration/deep-analysis-data-items-browser.test.ts b/tests/integration/deep-analysis-data-items-browser.test.ts new file mode 100644 index 00000000..6a1cc2b8 --- /dev/null +++ b/tests/integration/deep-analysis-data-items-browser.test.ts @@ -0,0 +1,105 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { type Browser, type Page, chromium } from 'playwright'; +import { WithDeepAnalysis } from '../../src/ai/researcher/deep-analysis.ts'; + +const DeepAnalysis = WithDeepAnalysis(Object as any) as any; + +const STATE = { url: '/list', hash: '', html: '', ariaSnapshot: '' }; + +function dataSection(container: string): string { + return `## Data: Items\n\n> Container: \`${container}\`\n\nItems, 2 items.\n`; +} + +describe('deep analysis of Data sections', () => { + let browser: Browser; + let page: Page; + + beforeAll(async () => { + browser = await chromium.launch(); + page = await browser.newPage(); + }); + + afterAll(async () => { + await browser.close(); + }); + + async function attemptedClicks(html: string, research: string): Promise { + await page.setContent(html); + const attempted: string[] = []; + const analysis = new DeepAnalysis(); + analysis.config = { ai: {} }; + analysis.provider = { getModelForAgent: () => null, chat: async () => ({ text: '' }), hasVision: () => false }; + analysis.stateManager = { getCurrentState: () => STATE }; + analysis.navigateTo = async () => {}; + analysis.cancelInUi = async () => {}; + analysis.explorer = { + withPage: async (fn: any) => fn(page), + capture: async () => {}, + action: () => ({ + attempt: async (code: string) => { + if (code.startsWith('I.click')) attempted.push(code); + return false; + }, + }), + }; + await analysis.performDeepAnalysis(STATE, { text: research }); + return attempted; + } + + it('opens one item when the item declares a popup', async () => { + const clicks = await attemptedClicks( + `
+
First
+
Second
+
`, + dataSection('#grid') + ); + + expect(clicks.length).toBeGreaterThan(0); + expect(clicks.some((code) => code.includes('View first item'))).toBe(true); + expect(clicks.some((code) => code.includes('View second item'))).toBe(false); + }); + + it('follows the first link of a list', async () => { + const clicks = await attemptedClicks( + ``, + dataSection('#list') + ); + + expect(clicks.some((code) => code.includes('First record'))).toBe(true); + expect(clicks.some((code) => code.includes('Second record'))).toBe(false); + }); + + it('never presses an item control that does not declare where it leads', async () => { + const clicks = await attemptedClicks( + ` + + +
First
Second
`, + dataSection('#rows') + ); + + expect(clicks).toEqual([]); + }); + + it('skips a safe control when it sits after an unsafe one', async () => { + const clicks = await attemptedClicks( + ``, + dataSection('#list') + ); + + expect(clicks.some((code) => code.includes('Archive'))).toBe(false); + expect(clicks.some((code) => code.includes('First record'))).toBe(true); + }); + + it('ignores a Data section whose container is not on the page', async () => { + const clicks = await attemptedClicks('Elsewhere', dataSection('#gone')); + + expect(clicks).toEqual([]); + }); +});