diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 4b9d5d3d..b59d0bab 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -26,11 +26,8 @@ on: scenario: description: Scenario to run type: choice - options: [all, basic, experience, slow, gated, staged-login, variants] + options: [all, basic, experience, slow, gated, gated-no-auth, staged-login, variants] default: all - retries: - description: Retries per scenario - default: '2' variant: description: Widget variant type: choice @@ -69,7 +66,7 @@ jobs: run: bunx playwright install --with-deps chromium - name: Run regression scenarios - run: bunx bunosh regression:${{ github.event.inputs.scenario || 'all' }} --retries ${{ github.event.inputs.retries || '2' }} --variant ${{ github.event.inputs.variant || 'native' }} --seed ${{ github.event.inputs.seed || '42' }} + run: bunx bunosh regression:${{ github.event.inputs.scenario || 'all' }} --variant ${{ github.event.inputs.variant || 'native' }} --seed ${{ github.event.inputs.seed || '42' }} env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} diff --git a/Bunoshfile.js b/Bunoshfile.js index 36dbfd88..189438f4 100644 --- a/Bunoshfile.js +++ b/Bunoshfile.js @@ -8,7 +8,7 @@ const highlight = require('cli-highlight').highlight; import matter from 'gray-matter'; import yaml from 'js-yaml'; import { chromium } from 'playwright'; -import { assertBasicRun, assertControlRun, assertNavigatedRun, assertSeededRun, status } from './tests/regression/lib/assertions.ts'; +import { assertBasicRun, assertBlockedRun, assertControlRun, assertNavigatedRun, assertSeededRun, status } from './tests/regression/lib/assertions.ts'; import { readSessionAnalysis } from './tests/regression/lib/artifacts.ts'; import { DISCUSSION_CATEGORY, DISCUSSION_MUTATION, DISCUSSION_QUERY, prNumberFromEvent, repoOwnerAndName } from './tests/regression/lib/github.ts'; import { REPORT_MARKER, buildReport, fmtDuration } from './tests/regression/lib/report.ts'; @@ -23,11 +23,12 @@ const { shell, writeToFile, task, stopOnFail, ai } = global.bunosh; const CLI = resolve('bin/explorbot-cli.ts'); const REG_ROOT = resolve('tests/regression'); const SLOW_START_MS = 12000; -const BASIC_SCENARIO = { name: 'basic', path: '/issues', knowledge: 'login.md', fixture: {}, maxTests: 5, run(runDir) { return shell`timeout 900 bun ${CLI} explore ${this.path} --headless --max-tests ${this.maxTests} -p ${runDir}`; }, assert(runDir, stdout) { return assertBasicRun(runDir, stdout, this.minPassed); } }; +const BASIC_SCENARIO = { name: 'basic', path: '/issues', knowledge: 'login.md', fixture: {}, maxTests: 5, timeoutSec: 900, run(runDir) { return shell`timeout ${this.timeoutSec} bun ${CLI} explore ${this.path} --headless --max-tests ${this.maxTests} -p ${runDir}`; }, assert(runDir, stdout) { return assertBasicRun(runDir, stdout, this.minPassed); } }; const SLOW_SCENARIO = { ...BASIC_SCENARIO, name: 'slow', fixture: { slowStartMs: SLOW_START_MS }, maxTests: 1, minPassed: 1 }; const GATED_SCENARIO = { ...BASIC_SCENARIO, name: 'gated', path: '/workspace', knowledge: 'workspace.md', maxTests: 1, minPassed: 1 }; +const GATED_NO_AUTH_SCENARIO = { ...GATED_SCENARIO, name: 'gated-no-auth', knowledge: null, timeoutSec: 180, assert: (_runDir, stdout, finished) => assertBlockedRun(stdout, finished, 180) }; const STAGED_LOGIN_SCENARIO = { name: 'staged-login', path: '/account', knowledge: 'staged-login.md', fixture: {}, run(runDir) { return shell`timeout 300 bun ${CLI} navigate ${this.path} --headless -p ${runDir}`; }, assert: (_runDir, stdout, finished) => assertNavigatedRun(stdout, finished) }; -const SCENARIO_ORDER = ['basic', 'experience', 'slow', 'gated', 'staged-login']; +const SCENARIO_ORDER = ['basic', 'experience', 'slow', 'gated', 'staged-login', 'gated-no-auth']; const HISTORY_RUNS = 5; const SUMMARY_FILE = 'records.json'; const RUNS = join(REG_ROOT, '.runs'); @@ -323,59 +324,64 @@ export async function htmlAccessibility(filename) { /** * Regression scenario A: fresh explore of the Trackly fixture with real AI. * @param {object} options - * @param {number} [options.retries=2] - Re-runs after a failed attempt * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant */ -export async function regressionBasic(options = { retries: 2, variant: 'native', seed: 42 }) { +export async function regressionBasic(options = { variant: 'native', seed: 42 }) { await gateBasicScenario(options, BASIC_SCENARIO); } /** * Regression scenario C: explore a start page that renders client-side long after load. * @param {object} options - * @param {number} [options.retries=2] - Re-runs after a failed attempt * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant */ -export async function regressionSlow(options = { retries: 2, variant: 'native', seed: 42 }) { +export async function regressionSlow(options = { variant: 'native', seed: 42 }) { await gateBasicScenario(options, SLOW_SCENARIO); } /** * Regression scenario D: explore a page that shows a sign-in form at its own URL instead of redirecting. * @param {object} options - * @param {number} [options.retries=2] - Re-runs after a failed attempt * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant */ -export async function regressionGated(options = { retries: 2, variant: 'native', seed: 42 }) { +export async function regressionGated(options = { variant: 'native', seed: 42 }) { await gateBasicScenario(options, GATED_SCENARIO); } /** * Regression scenario E: the Navigator reaches a page behind a sign-in split over three pages: email, password, one-time code. * @param {object} options - * @param {number} [options.retries=2] - Re-runs after a failed attempt * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant */ -export async function regressionStagedLogin(options = { retries: 2, variant: 'native', seed: 42 }) { +export async function regressionStagedLogin(options = { variant: 'native', seed: 42 }) { await gateBasicScenario(options, STAGED_LOGIN_SCENARIO); } +/** + * Regression scenario F: the same sign-in page as scenario D with no credentials. Explorbot must give up on its own before the timeout, handing the blocker to Captain at most once. + * @param {object} options + * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random + * @param {number} [options.seed=42] - RNG seed for the random variant + */ +export async function regressionGatedNoAuth(options = { variant: 'native', seed: 42 }) { + await gateBasicScenario(options, GATED_NO_AUTH_SCENARIO); +} + /** * Regression scenario B: experience reuse. Control run must fail without seeds; seeded run must pass. * @param {object} options - * @param {number} [options.retries=2] - Re-runs of the seeded direction * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant */ -export async function regressionExperience(options = { retries: 2, variant: 'native', seed: 42 }) { +export async function regressionExperience(options = { variant: 'native', seed: 42 }) { if (!(await requireKey())) return; const variant = validateVariant(options.variant); if (!variant) return; - const result = await runExperienceScenario(variant, Number(options.seed) || 42, Number(options.retries) || 0); + const result = await runExperienceScenario(variant, Number(options.seed) || 42); writeReportFiles(); await gate('Scenario experience: control', result.controlPassed, 'control run passed without seeds — vault gate is broken'); await gate('Scenario experience: seeded', result.seededPassed, 'seeded run failed to pass'); @@ -384,26 +390,23 @@ export async function regressionExperience(options = { retries: 2, variant: 'nat /** * Run all regression scenarios in parallel. * @param {object} options - * @param {number} [options.retries=2] * @param {string} [options.variant=native] * @param {number} [options.seed=42] */ -export async function regressionAll(options = { retries: 2, variant: 'native', seed: 42 }) { - await Promise.all([regressionBasic(options), regressionExperience(options), regressionSlow(options), regressionGated(options), regressionStagedLogin(options)]); +export async function regressionAll(options = { variant: 'native', seed: 42 }) { + await Promise.all([regressionBasic(options), regressionExperience(options), regressionSlow(options), regressionGated(options), regressionStagedLogin(options), regressionGatedNoAuth(options)]); } /** * Informational variant matrix. Runs a scenario across variants without gating on assertion outcomes. * @param {object} options - * @param {number} [options.retries=0] * @param {string} [options.scenario=basic] - basic|experience * @param {string} [options.variants=native,aria,plain] - comma-separated variant list * @param {number} [options.seed=42] */ -export async function regressionVariants(options = { retries: 0, scenario: 'basic', variants: 'native,aria,plain', seed: 42 }) { +export async function regressionVariants(options = { scenario: 'basic', variants: 'native,aria,plain', seed: 42 }) { if (!(await requireKey())) return; const seed = Number(options.seed) || 42; - const retries = Number(options.retries) || 0; const scenario = options.scenario || 'basic'; const variants = String(options.variants || 'native,aria,plain').split(',').map((v) => v.trim()).filter(Boolean); for (const variant of variants) { @@ -411,8 +414,8 @@ export async function regressionVariants(options = { retries: 0, scenario: 'basi yell(`Skipping invalid variant: ${variant}`); continue; } - if (scenario === 'experience') await runExperienceScenario(variant, seed, retries); - else await runBasicScenario(variant, seed, retries); + if (scenario === 'experience') await runExperienceScenario(variant, seed); + else await runBasicScenario(variant, seed); } writeReportFiles(); } @@ -572,7 +575,7 @@ export async function regressionSmoke() { * Records per-request cost, tokens and latency; archives each run under tests/regression/.bench. * @param {string} [model] - OpenRouter model slug, e.g. inception/mercury-2.5 * @param {object} options - * @param {number} [options.runs=2] - Runs per model (no retries: every run counts) + * @param {number} [options.runs=2] - Runs per model * @param {string} [options.slot=base] - Slot to swap: base|agentic|vision * @param {string} [options.variant=native] - Widget variant: native|aria|plain|random * @param {number} [options.seed=42] - RNG seed for the random variant @@ -602,12 +605,12 @@ export async function benchModel(model = '', options = { runs: 2, slot: 'base', const index = records.length; const treeBefore = srcFingerprint(); - await runBasicScenario(variant, seed, 0); + await runBasicScenario(variant, seed); const treeAfter = srcFingerprint(); if (treeBefore !== treeAfter) yell(`src/ changed during run ${run} — this run is not comparable to the others`); const record = records[index] || {}; - const runDir = join(RUNS, `${dirName('basic', variant, seed)}-a1`); + const runDir = join(RUNS, dirName('basic', variant, seed)); if (existsSync(runDir)) cpSync(runDir, join(dir, 'run'), { recursive: true }); writeFileSync(join(dir, 'meta.json'), JSON.stringify({ model: slug, label, slot: options.slot || 'base', run, variant, seed, tree: treeAfter, treeStable: treeBefore === treeAfter, passed: record.passed, durationSec: record.durationSec, details: record.details }, null, 2)); } @@ -664,58 +667,46 @@ async function gateBasicScenario(options, scenario) { if (!(await requireKey())) return; const variant = validateVariant(options.variant); if (!variant) return; - const result = await runBasicScenario(variant, Number(options.seed) || 42, Number(options.retries) || 0, scenario); + const passed = await runBasicScenario(variant, Number(options.seed) || 42, scenario); writeReportFiles(); - await gate(`Scenario ${scenario.name} (${variant})`, result.passed, `${scenario.name} (${variant}) failed after ${result.attempts} attempt(s)`); + await gate(`Scenario ${scenario.name} (${variant})`, passed, `${scenario.name} (${variant}) failed`); } -async function runBasicScenario(variant, seed, retries, scenario = BASIC_SCENARIO) { +async function runBasicScenario(variant, seed, scenario = BASIC_SCENARIO) { const label = `${scenario.name} (${variant})`; - const maxAttempts = retries + 1; - let passed = false; - let attempts = 0; - for (let attempt = 1; attempt <= maxAttempts && !passed; attempt++) { - attempts = attempt; - const fixture = startFixture({ variant, seed, ...scenario.fixture }); - const runDir = prepareRunDir(dirName(scenario.name, variant, seed), attempt); - cpSync(join(SEEDS, 'knowledge', scenario.knowledge), join(runDir, 'knowledge', scenario.knowledge)); - const started = Date.now(); - const res = await scenario.run(runDir).env(runEnv(fixture.url)); - fixture.stop(); - const outcome = scenario.assert(runDir, String(res.output || ''), !res.hasFailed); - recordAttempt(runDir, { scenario: scenario.name, label, kind: 'gate', attempt, maxAttempts, passed: outcome.passed, durationSec: elapsed(started), details: outcome.details, analysis: readSessionAnalysis(runDir) }); - reportAttempt(label, attempt, maxAttempts, outcome); - passed = outcome.passed; - } - return { passed, attempts }; + const fixture = startFixture({ variant, seed, ...scenario.fixture }); + const runDir = prepareRunDir(dirName(scenario.name, variant, seed)); + if (scenario.knowledge) cpSync(join(SEEDS, 'knowledge', scenario.knowledge), join(runDir, 'knowledge', scenario.knowledge)); + const started = Date.now(); + const res = await scenario.run(runDir).env(runEnv(fixture.url)); + fixture.stop(); + const outcome = scenario.assert(runDir, String(res.output || ''), !res.hasFailed); + recordRun(runDir, { scenario: scenario.name, label, kind: 'gate', passed: outcome.passed, durationSec: elapsed(started), details: outcome.details, analysis: readSessionAnalysis(runDir) }); + reportRun(label, outcome); + return outcome.passed; } -async function runExperienceScenario(variant, seed, retries) { +async function runExperienceScenario(variant, seed) { const controlFixture = startFixture({ variant, seed }); - const controlDir = prepareRunDir(dirName('exp-control', variant, seed), 1); + const controlDir = prepareRunDir(dirName('exp-control', variant, seed)); const controlStart = Date.now(); const controlRes = await shell`timeout 720 bun ${CLI} test ${VAULT_PLAN} all --headless -p ${controlDir}`.env(runEnv(controlFixture.url)); controlFixture.stop(); const control = assertControlRun(controlDir, !controlRes.hasFailed); - recordAttempt(controlDir, { scenario: 'experience', label: 'experience: control', kind: 'control', attempt: 1, maxAttempts: 1, passed: control.passed, durationSec: elapsed(controlStart), details: control.details }); - reportAttempt('experience: control', 1, 1, control); - - const maxAttempts = retries + 1; - let seededPassed = false; - for (let attempt = 1; attempt <= maxAttempts && !seededPassed; attempt++) { - const fixture = startFixture({ variant, seed }); - const runDir = prepareRunDir(dirName('exp-seeded', variant, seed), attempt); - cpSync(join(SEEDS, 'experience'), join(runDir, 'experience'), { recursive: true }); - cpSync(join(SEEDS, 'knowledge', 'vault.md'), join(runDir, 'knowledge', 'vault.md')); - const started = Date.now(); - const res = await shell`timeout 720 bun ${CLI} test ${VAULT_PLAN} all --headless -p ${runDir}`.env(runEnv(fixture.url)); - fixture.stop(); - const outcome = assertSeededRun(runDir); - recordAttempt(runDir, { scenario: 'experience', label: 'experience: seeded', kind: 'gate', attempt, maxAttempts, passed: outcome.passed, durationSec: elapsed(started), details: outcome.details }); - reportAttempt('experience: seeded', attempt, maxAttempts, outcome); - seededPassed = outcome.passed; - } - return { controlPassed: control.passed, seededPassed }; + recordRun(controlDir, { scenario: 'experience', label: 'experience: control', kind: 'control', passed: control.passed, durationSec: elapsed(controlStart), details: control.details }); + reportRun('experience: control', control); + + const fixture = startFixture({ variant, seed }); + const runDir = prepareRunDir(dirName('exp-seeded', variant, seed)); + cpSync(join(SEEDS, 'experience'), join(runDir, 'experience'), { recursive: true }); + cpSync(join(SEEDS, 'knowledge', 'vault.md'), join(runDir, 'knowledge', 'vault.md')); + const started = Date.now(); + await shell`timeout 720 bun ${CLI} test ${VAULT_PLAN} all --headless -p ${runDir}`.env(runEnv(fixture.url)); + fixture.stop(); + const seeded = assertSeededRun(runDir); + recordRun(runDir, { scenario: 'experience', label: 'experience: seeded', kind: 'gate', passed: seeded.passed, durationSec: elapsed(started), details: seeded.details }); + reportRun('experience: seeded', seeded); + return { controlPassed: control.passed, seededPassed: seeded.passed }; } @@ -806,8 +797,8 @@ function cachePercent(cached, prompt) { return `${Math.round((cached / prompt) * 100)}%`; } -function prepareRunDir(name, attempt) { - const dir = join(RUNS, `${name}-a${attempt}`); +function prepareRunDir(name) { + const dir = join(RUNS, name); rmSync(dir, { recursive: true, force: true }); mkdirSync(join(dir, 'knowledge'), { recursive: true }); mkdirSync(join(dir, 'experience'), { recursive: true }); @@ -815,7 +806,7 @@ function prepareRunDir(name, attempt) { return dir; } -function recordAttempt(runDir, record) { +function recordRun(runDir, record) { records.push(record); writeFileSync(join(runDir, 'record.json'), JSON.stringify(record)); } @@ -858,9 +849,8 @@ function writeReportFiles(list = records, history = []) { if (summaryPath) writeFileSync(summaryPath, `${markdown}\n`, { flag: 'a' }); } -function reportAttempt(label, attempt, maxAttempts, outcome) { - const verdict = outcome.passed ? 'PASS' : 'FAIL'; - say(`${label} attempt ${attempt}/${maxAttempts}: ${verdict}`); +function reportRun(label, outcome) { + say(`${label}: ${status(outcome.passed)}`); for (const detail of outcome.details) say(` ${detail}`); } diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ed049c7..1d585e79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,18 @@ - [Planner] A Scout failure no longer stops planning. The log shows a warning and the plan is made without documentation. - Prima: `npx prima-cli` releases are published again. Nothing newer than 0.5.4 reached npm because a release test failed, so fixes such as treating `localhost` and `127.0.0.1` as one site were missing from it. +## 2026-11-01 + +### Changes +- [Explore] A page that cannot be opened (for example a sign-in form with no credentials) is now skipped as a whole: no more planning styles or tests run on it, and pages that look the same are skipped without trying again, listed as "matches a blocked page". Exploration stops after 3 such pages. +- [Explore] When the start page itself is blocked, the run ends right away and suggests how to unblock it: pass credentials with `--auth`, or save them with `learn`. If knowledge for the page exists but did not help, it suggests updating it. + ```bash + explorbot explore /workspace --auth 'login: admin@example.com password: secret123' + explorbot learn /workspace "login: admin@example.com password: secret123" + ``` +- [Captain] Takes over a blocked page once. It no longer gets the same blocked page again each time the Planner words the problem differently. +- Console output: blocker messages stand out with an empty line before and after, and empty lines no longer stack up. + ## 2026-10-31 ### New CLI Options diff --git a/boat/doc-collector/src/template-dedup.ts b/boat/doc-collector/src/template-dedup.ts index 84ccafe3..690813af 100644 --- a/boat/doc-collector/src/template-dedup.ts +++ b/boat/doc-collector/src/template-dedup.ts @@ -1,5 +1,5 @@ import { ariaTemplateSignature } from '../../../src/utils/aria.ts'; -import { diceSimilarity } from '../../../src/utils/similarity.ts'; +import { similarityScore } from '../../../src/utils/similarity.ts'; const TEMPLATE_MIN_SIGNATURE_SIZE = 8; const TEMPLATE_SIMILARITY_THRESHOLD = 90; @@ -16,7 +16,7 @@ export function findTemplateMatch(snapshot: string | null, known: TemplateRecord let bestUrl: string | null = null; let bestScore = 0; for (const record of known) { - const score = diceSimilarity(signature, record.signature); + const score = similarityScore(signature, record.signature); if (score < bestScore) continue; bestScore = score; bestUrl = record.url; diff --git a/docs/contributing/regression-tests.md b/docs/contributing/regression-tests.md index b11e033e..80204bc4 100644 --- a/docs/contributing/regression-tests.md +++ b/docs/contributing/regression-tests.md @@ -40,9 +40,13 @@ Runs scenario A's assertions against `explore /workspace`. Signed out, `/workspa Runs `explorbot navigate /account` with the credentials in `seeds/knowledge/staged-login.md`. Signed out, `/account` redirects to `/signin`, which asks only for the email; `/signin/password` asks only for the password and `/signin/otp` only for the one-time code. The later fields do not exist until the earlier stage is submitted, so the Navigator has to read each page as it arrives. The run must exit 0 with no Captain handover, since a Captain rescue would hide a Navigator failure. It tests navigation only: no research, planning or test execution. +### Scenario F — sign-in form without credentials (`regression:gated-no-auth`) + +Runs `explore /workspace` with no knowledge, so nothing can sign in. Explorbot must give up on its own: the run fails if it is still going after 180 seconds, or if Captain takes over the blocked page more than once (`MAX_CAPTAIN_HANDOVERS` in `tests/regression/lib/assertions.ts`). + ## The fixture app — "Trackly" -A self-contained issue tracker served in-process by `tests/regression/fixture/server.ts` (`Bun.serve`, ephemeral port, fresh in-memory store per attempt). No external network, no database. +A self-contained issue tracker served in-process by `tests/regression/fixture/server.ts` (`Bun.serve`, ephemeral port, fresh in-memory store per run). No external network, no database. | Route | Auth | Contents | |-------|------|----------| @@ -81,15 +85,16 @@ bunx bunosh regression:serve # Real-AI runs (need a provider key) export OPENROUTER_API_KEY=sk-... -bunx bunosh regression:experience --retries 0 # cheapest AI path -bunx bunosh regression:basic --retries 0 -bunx bunosh regression:slow --retries 0 -bunx bunosh regression:gated --retries 0 -bunx bunosh regression:staged-login --retries 0 -bunx bunosh regression:all # all scenarios, default retries +bunx bunosh regression:experience # cheapest AI path +bunx bunosh regression:basic +bunx bunosh regression:slow +bunx bunosh regression:gated +bunx bunosh regression:staged-login +bunx bunosh regression:gated-no-auth +bunx bunosh regression:all # all scenarios ``` -Each attempt runs in an isolated throwaway directory under `tests/regression/.runs/` (gitignored). Inspect `tests/regression/.runs/--a/output/` for the research, plans, and reporter markdown a run produced, and `tests/regression/.runs/report.md` for the summary. +Every scenario runs once; there are no retries, so a flaky pass or fail shows up as it happened. Each run uses an isolated throwaway directory under `tests/regression/.runs/` (gitignored). Inspect `tests/regression/.runs/-/output/` for the research, plans, and reporter markdown a run produced, and `tests/regression/.runs/report.md` for the summary. ### Variant matrix @@ -103,11 +108,11 @@ bunx bunosh regression:variants --scenario basic --variants native,aria,plain `.github/workflows/regression.yml` runs on `pull_request` and `workflow_dispatch`. `regression:all` runs the scenarios in parallel inside one job, so a run lasts as long as its slowest scenario and needs one approval. The job uses the `regression` GitHub environment, which holds `OPENROUTER_API_KEY` behind a **required reviewer**. Every run — including fork PRs — pauses at the environment gate before the secret is exposed. The job runs the PR's own code, so the reviewer must read the PR diff (workflow, Bunoshfile, fixture, lib) before approving; that review is the security boundary. The workflow never uses `pull_request_target`. -Every attempt writes a `record.json` into its run directory. `regression:report` merges those records and posts the report as a sticky PR comment (on `pull_request`) or a `Regression Reports` GitHub Discussion (on `workflow_dispatch`). Fork PRs get a read-only token, so comment posting may fail; the report is also written to the step summary. +Every run writes a `record.json` into its run directory. `regression:report` merges those records and posts the report as a sticky PR comment (on `pull_request`) or a `Regression Reports` GitHub Discussion (on `workflow_dispatch`). Fork PRs get a read-only token, so comment posting may fail; the report is also written to the step summary. ### Speed comparison -The report's `Time` column is the duration of the passing attempt. `Previous median` is the median of that scenario's passing time over the last 5 runs, and `Change` flags a difference of 20% or more. The job uploads the run's records as the `regression-summary` artifact (the run directories go to `regression-runs`) (kept 90 days), and the next run downloads the latest five to compare against. Which tests the planner picks changes a run's time, so read one flagged run as a hint and a trend over several as a regression. +The report's `Time` column is the duration of the run. `Previous median` is the median of that scenario's passing time over the last 5 runs, and `Change` flags a difference of 20% or more. The job uploads the run's records as the `regression-summary` artifact (the run directories go to `regression-runs`) (kept 90 days), and the next run downloads the latest five to compare against. Which tests the planner picks changes a run's time, so read one flagged run as a hint and a trend over several as a regression. The report also embeds the **Session Analysis** from the fresh-explore run — Explorbot's Analyst agent writes a prose summary of what works, defects, UX issues, and execution issues, which the harness reads from `output/reports/