diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index b4c7de6b..4b9d5d3d 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -26,7 +26,7 @@ on: scenario: description: Scenario to run type: choice - options: [all, basic, experience, slow, gated, variants] + options: [all, basic, experience, slow, gated, staged-login, variants] default: all retries: description: Retries per scenario diff --git a/Bunoshfile.js b/Bunoshfile.js index 7a02a4d9..36dbfd88 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -8,11 +8,11 @@ const highlight = require('cli-highlight').highlight; import matter from 'gray-matter'; import yaml from 'js-yaml'; import { chromium } from 'playwright'; -import { assertBasicRun, assertControlRun, assertSeededRun, status } from './tests/regression/lib/assertions.ts'; +import { assertBasicRun, assertControlRun, assertNavigatedRun, assertSeededRun, status } from './tests/regression/lib/assertions.ts'; import { readSessionAnalysis } from './tests/regression/lib/artifacts.ts'; import { DISCUSSION_CATEGORY, DISCUSSION_MUTATION, DISCUSSION_QUERY, prNumberFromEvent, repoOwnerAndName } from './tests/regression/lib/github.ts'; import { REPORT_MARKER, buildReport, fmtDuration } from './tests/regression/lib/report.ts'; -import { startFixture } from './tests/regression/fixture/server.ts'; +import { SIGN_IN_OTP, startFixture } from './tests/regression/fixture/server.ts'; import { parsePlansFromMarkdown } from './src/utils/test-plan-markdown.ts'; import { htmlCombinedSnapshot, htmlTextSnapshot, minifyHtml } from './src/utils/html.js'; import { analyzeDemoCandidates, createDemoVideo } from './.claude/skills/demo-video/demo-video.ts'; @@ -23,10 +23,11 @@ const { shell, writeToFile, task, stopOnFail, ai } = global.bunosh; const CLI = resolve('bin/explorbot-cli.ts'); const REG_ROOT = resolve('tests/regression'); const SLOW_START_MS = 12000; -const BASIC_SCENARIO = { name: 'basic', path: '/issues', knowledge: 'login.md', fixture: {}, maxTests: 5 }; +const BASIC_SCENARIO = { name: 'basic', path: '/issues', knowledge: 'login.md', fixture: {}, maxTests: 5, run(runDir) { return shell`timeout 900 bun ${CLI} explore ${this.path} --headless --max-tests ${this.maxTests} -p ${runDir}`; }, assert(runDir, stdout) { return assertBasicRun(runDir, stdout, this.minPassed); } }; const SLOW_SCENARIO = { ...BASIC_SCENARIO, name: 'slow', fixture: { slowStartMs: SLOW_START_MS }, maxTests: 1, minPassed: 1 }; const GATED_SCENARIO = { ...BASIC_SCENARIO, name: 'gated', path: '/workspace', knowledge: 'workspace.md', maxTests: 1, minPassed: 1 }; -const SCENARIO_ORDER = ['basic', 'experience', 'slow', 'gated']; +const STAGED_LOGIN_SCENARIO = { name: 'staged-login', path: '/account', knowledge: 'staged-login.md', fixture: {}, run(runDir) { return shell`timeout 300 bun ${CLI} navigate ${this.path} --headless -p ${runDir}`; }, assert: (_runDir, stdout, finished) => assertNavigatedRun(stdout, finished) }; +const SCENARIO_ORDER = ['basic', 'experience', 'slow', 'gated', 'staged-login']; const HISTORY_RUNS = 5; const SUMMARY_FILE = 'records.json'; const RUNS = join(REG_ROOT, '.runs'); @@ -352,6 +353,17 @@ export async function regressionGated(options = { retries: 2, variant: 'native', await gateBasicScenario(options, GATED_SCENARIO); } +/** + * Regression scenario E: the Navigator reaches a page behind a sign-in split over three pages: email, password, one-time code. + * @param {object} options + * @param {number} [options.retries=2] - Re-runs after a failed attempt + * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random + * @param {number} [options.seed=42] - RNG seed for the random variant + */ +export async function regressionStagedLogin(options = { retries: 2, variant: 'native', seed: 42 }) { + await gateBasicScenario(options, STAGED_LOGIN_SCENARIO); +} + /** * Regression scenario B: experience reuse. Control run must fail without seeds; seeded run must pass. * @param {object} options @@ -377,7 +389,7 @@ export async function regressionExperience(options = { retries: 2, variant: 'nat * @param {number} [options.seed=42] */ export async function regressionAll(options = { retries: 2, variant: 'native', seed: 42 }) { - await Promise.all([regressionBasic(options), regressionExperience(options), regressionSlow(options), regressionGated(options)]); + await Promise.all([regressionBasic(options), regressionExperience(options), regressionSlow(options), regressionGated(options), regressionStagedLogin(options)]); } /** @@ -512,6 +524,36 @@ export async function regressionSmoke() { const workspace = await (await fetch(`${gatedFixture.url}/workspace`, { headers: { cookie: gatedCookie } })).text(); await check('signed-in /workspace renders the issue list', workspace.includes(' fetch(`${stagedFixture.url}${path}`, { method: 'POST', redirect: 'manual', headers: { cookie }, body: new URLSearchParams(fields) }); + const cookieOf = (response) => (response.headers.get('set-cookie') || '').split(';')[0]; + const stagedAnon = await fetch(`${stagedFixture.url}/account`, { redirect: 'manual' }); + await check('anonymous /account redirects to /signin', stagedAnon.status === 302 && stagedAnon.headers.get('location') === '/signin'); + const skippedToPassword = await fetch(`${stagedFixture.url}/signin/password`, { redirect: 'manual' }); + await check('password stage needs the email stage', skippedToPassword.headers.get('location') === '/signin'); + const skippedToOtp = await fetch(`${stagedFixture.url}/signin/otp`, { redirect: 'manual' }); + await check('code stage needs the password stage', skippedToOtp.headers.get('location') === '/signin'); + const emailStage = await stagedPost('/signin', { email: 'demo@example.com' }); + const emailCookie = cookieOf(emailStage); + await check('email stage leads to the password stage', emailStage.headers.get('location') === '/signin/password'); + const badPassword = await stagedPost('/signin/password', { password: 'nope' }, emailCookie); + await check('password stage rejects a wrong password', badPassword.status === 401); + const passwordStage = await stagedPost('/signin/password', { password: 'hunter2-fixture' }, emailCookie); + const pendingCookie = cookieOf(passwordStage); + await check('password stage leads to the code stage', passwordStage.headers.get('location') === '/signin/otp'); + const pendingAccount = await fetch(`${stagedFixture.url}/account`, { redirect: 'manual', headers: { cookie: pendingCookie } }); + await check('password alone does not open /account', pendingAccount.headers.get('location') === '/signin'); + const badCode = await stagedPost('/signin/otp', { code: '000000' }, pendingCookie); + await check('code stage rejects a wrong code', badCode.status === 401); + const codeStage = await stagedPost('/signin/otp', { code: SIGN_IN_OTP }, pendingCookie); + await check('code stage leads to /account', codeStage.headers.get('location') === '/account'); + const account = await (await fetch(`${stagedFixture.url}/account`, { headers: { cookie: cookieOf(codeStage) } })).text(); + await check('signed-in /account renders', account.includes('

Account

')); + stagedFixture.stop(); + const stagedKnowledge = matter(readFileSync(join(SEEDS, 'knowledge', 'staged-login.md'), 'utf-8')); + await check('staged sign-in knowledge targets /signin/* and carries the code', stagedKnowledge.data.url === '/signin/*' && stagedKnowledge.content.includes(SIGN_IN_OTP)); + const workspaceKnowledge = matter(readFileSync(join(SEEDS, 'knowledge', 'workspace.md'), 'utf-8')); await check('workspace knowledge targets /workspace', workspaceKnowledge.data.url === '/workspace'); @@ -638,9 +680,9 @@ async function runBasicScenario(variant, seed, retries, scenario = BASIC_SCENARI const runDir = prepareRunDir(dirName(scenario.name, variant, seed), attempt); cpSync(join(SEEDS, 'knowledge', scenario.knowledge), join(runDir, 'knowledge', scenario.knowledge)); const started = Date.now(); - const res = await shell`timeout 900 bun ${CLI} explore ${scenario.path} --headless --max-tests ${scenario.maxTests} -p ${runDir}`.env(runEnv(fixture.url)); + const res = await scenario.run(runDir).env(runEnv(fixture.url)); fixture.stop(); - const outcome = assertBasicRun(runDir, String(res.output || ''), scenario.minPassed); + const outcome = scenario.assert(runDir, String(res.output || ''), !res.hasFailed); recordAttempt(runDir, { scenario: scenario.name, label, kind: 'gate', attempt, maxAttempts, passed: outcome.passed, durationSec: elapsed(started), details: outcome.details, analysis: readSessionAnalysis(runDir) }); reportAttempt(label, attempt, maxAttempts, outcome); passed = outcome.passed; diff --git a/CHANGELOG.md b/CHANGELOG.md index b7092bab..03ce68f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## 2026-11-05 +### Changes +- [Navigator] Gets through flows spread over several pages, such as a sign-in that asks for the email, the password and a one-time code on separate pages. When a step moves the page on, Navigator now looks at the new page instead of spending its remaining attempts on the old one. + +## 2026-11-05 + ### New CLI Options - **`prima --version`** — Shows the installed prima version. Running `prima` with no arguments also prints it, above the help. ```bash diff --git a/docs/contributing/regression-tests.md b/docs/contributing/regression-tests.md index a9a04f6f..b11e033e 100644 --- a/docs/contributing/regression-tests.md +++ b/docs/contributing/regression-tests.md @@ -36,6 +36,10 @@ Runs scenario A again with the fixture's slow start on: on the first visit of a Runs scenario A's assertions against `explore /workspace`. Signed out, `/workspace` shows the sign-in form at its own URL instead of redirecting to `/login`, and signing in there keeps you on `/workspace`. The planner reports the page blocked by a sign-in form, so explore must hand the report to Captain, sign in with the credentials in `seeds/knowledge/workspace.md` and plan the page again. Like scenario C, it executes one test. +### Scenario E — sign-in split over three pages (`regression:staged-login`) + +Runs `explorbot navigate /account` with the credentials in `seeds/knowledge/staged-login.md`. Signed out, `/account` redirects to `/signin`, which asks only for the email; `/signin/password` asks only for the password and `/signin/otp` only for the one-time code. The later fields do not exist until the earlier stage is submitted, so the Navigator has to read each page as it arrives. The run must exit 0 with no Captain handover, since a Captain rescue would hide a Navigator failure. It tests navigation only: no research, planning or test execution. + ## The fixture app — "Trackly" A self-contained issue tracker served in-process by `tests/regression/fixture/server.ts` (`Bun.serve`, ephemeral port, fresh in-memory store per attempt). No external network, no database. @@ -49,6 +53,8 @@ A self-contained issue tracker served in-process by `tests/regression/fixture/se | `/settings` | yes | Profile/Preferences tabs + an activity iframe | | `/vault` | no | scenario B access-code gate | | `/workspace` | no / yes | scenario D: the sign-in form when signed out, the issue list when signed in | +| `/signin`, `/signin/password`, `/signin/otp` | no | scenario E: one sign-in stage per page, each refusing to open before the previous one is done | +| `/account` | yes | scenario E target: only the code stage signs in | | `/api/*` | cookie | REST API over the same store, `+ /api/openapi.json` | The REST API mirrors the store and is served for a future API-testing scenario; the current scenarios do not exercise it. @@ -79,6 +85,7 @@ bunx bunosh regression:experience --retries 0 # cheapest AI path bunx bunosh regression:basic --retries 0 bunx bunosh regression:slow --retries 0 bunx bunosh regression:gated --retries 0 +bunx bunosh regression:staged-login --retries 0 bunx bunosh regression:all # all scenarios, default retries ``` diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 834650a0..2fcde8fa 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -338,6 +338,7 @@ class Navigator implements Agent { if (freshHash !== prevHash && (attempt.ok || check.urlMatches)) { progressBlocks.push(codeBlock); } + if (freshHash !== prevHash) codeBlockIndex = codeBlocks.length; } else { resolved = attempt.ok; if (attempt.ok) progressBlocks.push(codeBlock); @@ -441,6 +442,8 @@ class Navigator implements Agent { contextMsg += `\n${lines}\n\n\n`; } + contextMsg += `\n${actionResult.toAiContext()}\n\n\n`; + if (includeHtml) { contextMsg += `Full HTML context:\n\n\n${await actionResult.combinedHtml()}\n\n\n`; } diff --git a/tests/regression/fixture/pages.ts b/tests/regression/fixture/pages.ts index 5a9a27a9..947552db 100644 --- a/tests/regression/fixture/pages.ts +++ b/tests/regression/fixture/pages.ts @@ -14,6 +14,12 @@ const PRIORITY_OPTIONS = [ { value: 'critical', label: 'Critical' }, ]; +const SIGN_IN_STAGES = { + email: { label: 'Email', name: 'email', type: 'email', hint: 'Step 1 of 3. Who is signing in?', action: '/signin', submit: 'Continue' }, + password: { label: 'Password', name: 'password', type: 'password', hint: 'Step 2 of 3. Enter your password.', action: '/signin/password', submit: 'Continue' }, + otp: { label: 'One-time code', name: 'code', type: 'text', hint: 'Step 3 of 3. Enter the one-time code.', action: '/signin/otp', submit: 'Verify' }, +}; + export function loginPage(error?: string, action = '/login'): string { const alert = error ? `` : ''; const body = ` @@ -30,6 +36,27 @@ export function loginPage(error?: string, action = '/login'): string { return doc('Sign in · Trackly', body, null, 'login'); } +export function signInStagePage(stage: keyof typeof SIGN_IN_STAGES, error?: string): string { + const { label, name, type, hint, action, submit } = SIGN_IN_STAGES[stage]; + let alert = ''; + if (error) alert = ``; + const body = ` +
+

${esc(label)}

+

${esc(hint)}

+ ${alert} +
+
+ +
+
`; + return doc(`${label} · Trackly`, body, null, 'login'); +} + +export function accountPage(user: PublicUser): string { + return doc('Account · Trackly', `

Account

Signed in as ${esc(user.name)}.

`, user, 'account'); +} + export function issuesPage(reg: Registry, store: Store, filters: { q?: string; status?: string; label?: string }, user: PublicUser, opts: { openDrawer?: boolean; error?: string } = {}): string { const labels = store.listLabels(); const users = store.listUsers(); diff --git a/tests/regression/fixture/server.ts b/tests/regression/fixture/server.ts index e5bb1968..7629b42b 100644 --- a/tests/regression/fixture/server.ts +++ b/tests/regression/fixture/server.ts @@ -1,9 +1,10 @@ import { handleApi } from './api.ts'; import { type Variant, type VariantMode, createRegistry } from './components.ts'; -import { activityWidget, issueDetailPage, issuesPage, loginPage, settingsPage, slowStartShell, vaultPage } from './pages.ts'; +import { accountPage, activityWidget, issueDetailPage, issuesPage, loginPage, settingsPage, signInStagePage, slowStartShell, vaultPage } from './pages.ts'; import { type Store, createStore } from './store.ts'; export const VAULT_CODE = 'K7-9284-XRAY-TANGO'; +export const SIGN_IN_OTP = '482913'; const VAULT_DOCS = ['Quarterly report', 'Design archive', 'Payroll ledger']; const SIGN_IN_TARGETS: Record = { '/login': '/issues', '/workspace': '/workspace' }; @@ -73,6 +74,29 @@ async function handle(req: Request, store: Store, defaultVariant: VariantMode, d if (path === '/workspace' && !user) return html(loginPage(undefined, '/workspace')); if (path === '/workspace') return html(issuesPage(reg, store, {}, user)); + if (path === '/signin' && method === 'GET') return html(signInStagePage('email')); + if (path === '/signin' && method === 'POST') { + const form = await req.formData(); + return redirect('/signin/password', `trackly_signin_email=${encodeURIComponent(String(form.get('email') || ''))}; Path=/`); + } + if (path === '/signin/password') { + if (!cookies.trackly_signin_email) return redirect('/signin'); + if (method === 'GET') return html(signInStagePage('password')); + const form = await req.formData(); + const token = store.login(decodeURIComponent(cookies.trackly_signin_email), String(form.get('password') || '')); + if (!token) return html(signInStagePage('password', 'Invalid email or password'), 401); + return redirect('/signin/otp', `trackly_signin_pending=${token}; Path=/; HttpOnly`); + } + if (path === '/signin/otp') { + if (!cookies.trackly_signin_pending) return redirect('/signin'); + if (method === 'GET') return html(signInStagePage('otp')); + const form = await req.formData(); + if (String(form.get('code') || '') !== SIGN_IN_OTP) return html(signInStagePage('otp', 'Invalid code'), 401); + return redirect('/account', `trackly_session=${cookies.trackly_signin_pending}; Path=/; HttpOnly`); + } + if (path === '/account' && !user) return redirect('/signin'); + if (path === '/account') return html(accountPage(user)); + if (!user) return redirect('/login'); if (path === '/') return redirect('/issues'); diff --git a/tests/regression/lib/assertions.ts b/tests/regression/lib/assertions.ts index 9da62e20..7a875582 100644 --- a/tests/regression/lib/assertions.ts +++ b/tests/regression/lib/assertions.ts @@ -54,6 +54,12 @@ export function assertBasicRun(runDir: string, stdout: string, minPassed = MIN_P return { passed: checks.every(Boolean), details }; } +export function assertNavigatedRun(stdout: string, finished: boolean): AssertResult { + const handovers = stdout.split('Captain takes over:').length - 1; + const details = [`reached target: ${status(finished)}`, `navigator alone: ${status(handovers === 0)} (captain handovers: ${handovers})`]; + return { passed: finished && handovers === 0, details }; +} + export function assertSeededRun(runDir: string): AssertResult { const summary = latestReporterSummary(runDir); if (!summary) return { passed: false, details: ['seeded: FAIL (no reporter summary written)'] }; diff --git a/tests/regression/seeds/knowledge/staged-login.md b/tests/regression/seeds/knowledge/staged-login.md new file mode 100644 index 00000000..f0e5ad49 --- /dev/null +++ b/tests/regression/seeds/knowledge/staged-login.md @@ -0,0 +1,9 @@ +--- +url: /signin/* +--- + +To sign in use these credentials: + +email: demo@example.com +password: hunter2-fixture +one-time code: 482913 diff --git a/tests/unit/navigator-resolve-state.test.ts b/tests/unit/navigator-resolve-state.test.ts index e8b28d68..1d5a171e 100644 --- a/tests/unit/navigator-resolve-state.test.ts +++ b/tests/unit/navigator-resolve-state.test.ts @@ -223,6 +223,25 @@ describe('Navigator resolveState', () => { expect(retry).toContain('Choose exactly ONE path'); }); + it('drops the rest of the batch once a step moves the page, and asks about the new page', async () => { + const stages = ['/signin/password', '/signin/otp', '/account']; + const harness = createHarness({ + responses: ["```js\nI.fillField('Email', 'a')\n```\n```js\nI.fillField('#email', 'a')\n```", "```js\nI.fillField('Password', 'b')\n```\n```js\nI.fillField('#password', 'b')\n```", "```js\nI.fillField('Code', 'c')\n```\n```js\nI.fillField('#code', 'c')\n```"], + ariaChanged: 'the next stage', + attempt: (_code, page) => { + page.url = stages.shift() as string; + page.hash = page.url; + return true; + }, + }); + + const resolved = await harness.navigator.resolveState('reach /account', fakeActionResult('/signin'), { expectedUrl: '/account' }); + + expect(resolved).toBe(true); + expect(harness.attempts).toEqual(["I.fillField('Email', 'a')", "I.fillField('Password', 'b')", "I.fillField('Code', 'c')"]); + expect(harness.sent.filter((text) => text.includes('the next stage'))).toHaveLength(2); + }); + it('asks for different strategies when nothing on the page reacted', async () => { const harness = createHarness({ responses: ["```js\nI.click('Sign in')\n```"], @@ -261,6 +280,39 @@ describe('Navigator resolveState', () => { expect(harness.sent[1]).toContain('menu opened'); }); + it('drops the rest of the batch when a step moves the page and a later line of it fails', async () => { + const harness = createHarness({ + responses: ["```js\nI.click('next')\nI.fillField('later', 'x')\n```\n```js\nI.click('stale')\n```", ''], + attempt: (_code, page) => { + page.url = '/next'; + page.hash = 'next'; + return false; + }, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.attempts).toHaveLength(1); + expect(harness.sent[1]).toContain('element not visible'); + }); + + it('shows every retry the page as it is now', async () => { + const pages = ['password page', 'code page']; + const harness = createHarness({ + responses: ["```js\nI.click('a')\n```", "```js\nI.click('b')\n```"], + attempt: (_code, page) => { + page.hash = pages[0]; + harness.action.actionResult = { ...fakeActionResult(), toAiContext: () => pages.shift() }; + return true; + }, + }); + + await harness.navigator.resolveState('reach /defects', fakeActionResult(), { expectedUrl: '/defects' }); + + expect(harness.sent[1]).toContain('\npassword page\n'); + expect(harness.sent[2]).toContain('\ncode page\n'); + }); + it('adds the full HTML context to the retry prompt only once', async () => { const harness = createHarness({ responses: ["```js\nI.click('a')\n```", "```js\nI.click('b')\n```", "```js\nI.click('c')\n```"],