From aa403accc4cd6cf49bd9e3bf9a7a3cf021980ce0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 9 Oct 2026 22:13:48 +0000 Subject: [PATCH 1/2] fix(engine): warn when tree-sitter parse fails A registered-language file that throws, returns no tree, or recovers as only ERROR nodes with no definitions now emits VG_WARN_PARSE_FAILED. The line names the path and language and tells you to fix the syntax or exclude the path. The same input always produces the same text; the parser exception is not part of the warning. vg scan prints that line when it builds the map. Closes #289 Signed-off-by: Cursor Agent Co-authored-by: vibgrate-team --- ARCHITECTURE.md | 5 +- CHANGELOG.md | 2 + DOCS.md | 2 +- src/engine/build.ts | 8 +- src/engine/parse-failure.ts | 28 +++++ src/engine/parse-worker.ts | 6 +- src/engine/parse.ts | 42 ++++++- src/engine/pool.ts | 6 +- src/reporting/commands/scan.ts | 11 ++ test/adversarial-fixtures.test.ts | 4 +- test/parse-failure.test.ts | 180 ++++++++++++++++++++++++++++++ 11 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 src/engine/parse-failure.ts create mode 100644 test/parse-failure.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 475150fb..6c8c34f5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -133,7 +133,10 @@ asserts stable output. 1. **build** discovers files, parses them via the worker pool using bundled tree-sitter WASM grammars, resolves symbols/imports into typed relations, - runs analysis, and writes a stable `graph.json`. + runs analysis, and writes a stable `graph.json`. A registered-language file + whose parse throws, returns no tree, or recovers as only ERROR nodes with + no definitions emits `VG_WARN_PARSE_FAILED` (path, language, recovery + hint). `vg scan` prints that same warning when it builds the map. 2. **query** commands and the **MCP server** both load `graph.json` and answer questions against it — they do not re-parse the source. 3. **drift reporting** runs on top of `@vibgrate/core-open` and the resolved diff --git a/CHANGELOG.md b/CHANGELOG.md index 887e092f..9712b117 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ backward compatible. ### Fixed +- **A supported-language file that fails tree-sitter parse warns with `VG_WARN_PARSE_FAILED`.** `vg build` and `vg scan` name the file path and the language, and tell you to fix the syntax or exclude the path with `--exclude`. The same file always produces the same warning. The map continues without that file's definitions. A parser crash is reported as that warning, not as a stack trace. (#289) + - **SBOM component order no longer follows scan or filesystem order.** `vg sbom export` and CycloneDX/SPDX graph export sort a component by its Package URL when one is written, otherwise by package name, then by version. diff --git a/DOCS.md b/DOCS.md index b6b753ce..7c37860b 100644 --- a/DOCS.md +++ b/DOCS.md @@ -5755,7 +5755,7 @@ A truncated or invalid lockfile stops the command. Warning codes cover condition | Code | Command | Meaning | | --- | --- | --- | -| `VG_WARN_PARSE_FAILED` | `vg build` | A source file failed to parse. The map continues without its symbols. | +| `VG_WARN_PARSE_FAILED` | `vg build`, `vg scan` | A supported-language file failed tree-sitter parse. The warning names the file and the language, and tells you to fix the syntax or exclude the path with `--exclude`. The map continues without that file's definitions. The parser's own exception text is not the warning. | | `VG_WARN_BUILD_FILE_OVERSIZE` | `vg build` | A file exceeded the per-file size cap and was left out of the map. | | `VG_WARN_TSC_RESOLVER_SKIPPED` | `vg build` | The TypeScript resolver was skipped because the corpus exceeded its file cap. | | `VG_WARN_YAML_PARSE_FAILED` | `vg build` | YAML for an infrastructure file failed to parse. | diff --git a/src/engine/build.ts b/src/engine/build.ts index 9252c40f..19a365da 100644 --- a/src/engine/build.ts +++ b/src/engine/build.ts @@ -51,6 +51,7 @@ import type { ResolveResult } from './resolve.js'; import { fileRolesFromParses } from './ast-roles.js'; import type { AstRoleHit } from '../core-open/scanners/architecture/ast-roles.js'; import { stampWarning, WARNING_CODES, type CodedWarning } from '../core-open/warnings.js'; +import { isParseFailedWarning } from './parse-failure.js'; import { assembleEngineWarnings } from './warning-codes.js'; export interface BuildOptions { @@ -308,8 +309,11 @@ export async function buildGraph(options: BuildOptions): Promise { // Never persist a failed parse: a transient parse-runtime crash (e.g. a // corrupted wasm heap) would otherwise poison the cache for that content // hash and every later build would reuse the empty parse instead of - // re-parsing the file. - if (p.defs.length === 0 && p.warnings?.some((w) => w.startsWith('parse failed:'))) continue; + // re-parsing the file. A syntax error that yielded no definitions is the + // same shape — re-parse it next time so a grammar fix is picked up, and + // so the warning is produced again from the file rather than from a + // cached empty table. + if (p.defs.length === 0 && p.warnings?.some((w) => isParseFailedWarning(w))) continue; const st = fileStats.find((f) => f.rel === p.rel); cache.set(p.rel, p, st ? { mtimeMs: st.mtimeMs, size: st.size } : undefined); } diff --git a/src/engine/parse-failure.ts b/src/engine/parse-failure.ts new file mode 100644 index 00000000..17dcd2c2 --- /dev/null +++ b/src/engine/parse-failure.ts @@ -0,0 +1,28 @@ +import { stampWarning, WARNING_CODES, warningPathLabel } from '../core-open/warnings.js'; +import { langById } from './languages.js'; + +/** + * Recovery sentence for {@link WARNING_CODES.PARSE_FAILED}. Stable: tests and + * CI assert this exact hint. It names no file contents and no exception text. + */ +export const PARSE_FAILURE_RECOVERY = 'Fix the syntax, or exclude the path with --exclude.'; + +/** + * Deterministic warning when a registered-language file fails tree-sitter + * parse. The path is repo-relative, the language is the registry label, and + * the hint is {@link PARSE_FAILURE_RECOVERY}. The parser's exception text is + * omitted — wasm failures often surface as a stack or a non-deterministic + * message, and that must not be the primary line. + */ +export function parseFailedWarning(rel: string, langId: string): string { + const language = langById(langId)?.label ?? langId; + return stampWarning( + WARNING_CODES.PARSE_FAILED, + `${warningPathLabel(rel)} (${language}): tree-sitter parse failed. The map continues without this file's definitions. ${PARSE_FAILURE_RECOVERY}`, + ); +} + +/** True for a stamped {@link WARNING_CODES.PARSE_FAILED} line, old or new. */ +export function isParseFailedWarning(warning: string): boolean { + return warning.endsWith(` [${WARNING_CODES.PARSE_FAILED}]`); +} diff --git a/src/engine/parse-worker.ts b/src/engine/parse-worker.ts index 47904988..c88a216c 100644 --- a/src/engine/parse-worker.ts +++ b/src/engine/parse-worker.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import { parseSource } from './parse.js'; import { setGrammarsOverride, resetParser } from './grammars.js'; import type { FileParse } from './types.js'; -import { stampWarning, WARNING_CODES } from '../core-open/warnings.js'; +import { parseFailedWarning } from './parse-failure.js'; /** * tinypool worker entry. Receives a chunk of files, reads and parses each, and @@ -32,7 +32,7 @@ export default async function run(payload: ParsePayload): Promise { try { const source = fs.readFileSync(task.abs, 'utf8'); out.push(await parseSource(task.rel, task.lang, source)); - } catch (err) { + } catch { // A wasm-level parse crash can leave the language's reused parser // mid-state; drop it so the failure stays contained to this file. resetParser(task.lang); @@ -47,7 +47,7 @@ export default async function run(payload: ParsePayload): Promise { heritage: [], typeRefs: [], guards: [], - warnings: [stampWarning(WARNING_CODES.PARSE_FAILED, `parse failed: ${(err as Error).message}`)], + warnings: [parseFailedWarning(task.rel, task.lang)], }); } } diff --git a/src/engine/parse.ts b/src/engine/parse.ts index 88c18391..d81c41f2 100644 --- a/src/engine/parse.ts +++ b/src/engine/parse.ts @@ -1,5 +1,6 @@ import { Query, type Node, type Language } from 'web-tree-sitter'; -import { parserFor, loadLanguage } from './grammars.js'; +import { parserFor, loadLanguage, resetParser } from './grammars.js'; +import { parseFailedWarning } from './parse-failure.js'; import { langById } from './languages.js'; import { queriesFor, type DefRule } from './queries.js'; import { extractEmbeddedScript } from './sfc.js'; @@ -287,8 +288,17 @@ export async function parseSource( const language = await loadLanguage(effLangId); const parser = await parserFor(def); - const tree = parser.parse(text); - if (!tree) return result; + let tree; + try { + tree = parser.parse(text); + } catch { + // An external scanner can throw (wasm) and leave the reused parser + // mid-state. Drop it so the next file of this language starts clean. + // The warning is stable: the exception text is not part of the line. + resetParser(effLangId); + return markParseFailed(result); + } + if (!tree) return markParseFailed(result); const root = tree.rootNode; // --- definitions --- @@ -467,7 +477,33 @@ export async function parseSource( const roles = extractAstRolesFromTree(rel, effLangId, language, root, text); if (roles) result.roles = roles; + // Decide before delete(): the node is invalid afterwards. A file whose root + // contains only ERROR nodes, and no definitions, used to disappear with no + // notice. A nested ERROR next to a real construct is not this case — some + // grammars set hasError on a reused parser for valid sources that simply + // declare nothing (Lua `return 1`), and warning there would be a false alarm. + const unrecovered = result.defs.length === 0 && unrecoveredTree(root); tree.delete(); + if (unrecovered) return markParseFailed(result); + return result; +} + +/** True when the parse recovered nothing but ERROR nodes at the root. */ +function unrecoveredTree(root: Node): boolean { + if (!root.hasError) return false; + if (root.isError || root.type === 'ERROR') return true; + let real = 0; + let error = 0; + for (const kid of root.namedChildren) { + if (!kid) continue; + if (kid.isError || kid.type === 'ERROR') error += 1; + else real += 1; + } + return error > 0 && real === 0; +} + +function markParseFailed(result: FileParse): FileParse { + result.warnings = [parseFailedWarning(result.rel, result.lang)]; return result; } diff --git a/src/engine/pool.ts b/src/engine/pool.ts index 9837770c..653e90b4 100644 --- a/src/engine/pool.ts +++ b/src/engine/pool.ts @@ -7,7 +7,7 @@ import { setGrammarsOverride, resetParser } from './grammars.js'; import { checkMemoryBudget, envJobs, envWorkerHeapMb, ResourceLimitError } from './limits.js'; import type { DiscoveredFile } from './discover.js'; import type { FileParse } from './types.js'; -import { stampWarning, WARNING_CODES } from '../core-open/warnings.js'; +import { parseFailedWarning } from './parse-failure.js'; import type { ParseTask } from './parse-worker.js'; /** @@ -78,11 +78,11 @@ async function parseInline(files: DiscoveredFile[], options: ParseOptions): Prom try { const source = fs.readFileSync(file.abs, 'utf8'); out.push(await parseSource(file.rel, file.lang.id, source)); - } catch (err) { + } catch { // A wasm-level parse crash can leave the language's reused parser // mid-state; drop it so the failure stays contained to this file. resetParser(file.lang.id); - out.push(emptyParse(file, stampWarning(WARNING_CODES.PARSE_FAILED, `parse failed: ${(err as Error).message}`))); + out.push(emptyParse(file, parseFailedWarning(file.rel, file.lang.id))); } onProgress?.(out.length, files.length); if (out.length % MEM_CHECK_EVERY === 0) checkMemoryBudget('parse', memoryBudgetMb); diff --git a/src/reporting/commands/scan.ts b/src/reporting/commands/scan.ts index f23006a0..22d532eb 100644 --- a/src/reporting/commands/scan.ts +++ b/src/reporting/commands/scan.ts @@ -18,6 +18,7 @@ import { loadConfig, findConfigFile, } from '../../core-open/index.js'; +import { formatWarningLine, WARNING_CODES, type CodedWarning } from '../../core-open/warnings.js'; import { writeScanSummary } from '../scan-summary.js'; import { compareDriftBudget, evaluateConfigDriftBudget } from '../drift-budget-gate.js'; import { @@ -694,6 +695,9 @@ export const scanCommand = new Command('scan') // Retained by the postScan hook so the reachability query below can run // against the freshly built map without a second (memory-heavy) build. let builtGraph: VgGraph | null = null; + // Parse failures from the code-map walk. Printed after the scan report so + // the progress renderer cannot overwrite them, and before any gate exit. + let parseFailureWarnings: CodedWarning[] = []; // Outcome of the infrastructure pack run (`--iac`). `null` means it never // ran: no code map, or the map build failed before it. Only an `ok` run // puts `extended.security` on the artifact — the section is never @@ -708,6 +712,7 @@ export const scanCommand = new Command('scan') exclude: opts.exclude, onParseProgress: (done, total) => report(done, total, 'parsing'), }); + parseFailureWarnings = result.codedWarnings.filter((warning) => warning.code === WARNING_CODES.PARSE_FAILED); builtGraph = result.graph; const written = writeArtifacts(result.graph, { root: rootDir }); if (written.architecturePolicyError) console.error(chalk.red(`\narchitecture policy: ${written.architecturePolicyError}`)); @@ -762,6 +767,12 @@ export const scanCommand = new Command('scan') const advanced = await loadAdvancedScanHook(); const artifact = await runCoreScan(rootDir, scanOpts, advanced); + // The map walk degrades per file. Surface the same stable line `vg build` + // prints so a scan does not omit a supported-language parse failure. + for (const warning of parseFailureWarnings) { + console.error(chalk.yellow(formatWarningLine(warning))); + } + // The scan just built a code map (its `postScan` step). Start the local // runtime if it is not up and hand it that map, so the very first `vg` in a // repo leaves behind a warm graph slot and a warming semantic index rather diff --git a/test/adversarial-fixtures.test.ts b/test/adversarial-fixtures.test.ts index 73d039d8..23adf9ca 100644 --- a/test/adversarial-fixtures.test.ts +++ b/test/adversarial-fixtures.test.ts @@ -96,9 +96,9 @@ for (const lang of buildableLangs) { // (c) no parse failures. `BuildResult.warnings` is the engine's only // per-file failure signal (parse/query problems surface as - // "parse failed: ..." warnings from the worker); it must be empty. + // VG_WARN_PARSE_FAILED); it must be empty. expect(first.warnings).toEqual([]); - expect(first.warnings.filter((w) => /query|parse failed/i.test(w))).toEqual([]); + expect(first.warnings.filter((w) => /query|VG_WARN_PARSE_FAILED|parse failed/i.test(w))).toEqual([]); // (d) determinism: rebuilding serializes byte-identically via the same // serializer the CLI writes graph.json with. For known re-parse-unstable diff --git a/test/parse-failure.test.ts b/test/parse-failure.test.ts new file mode 100644 index 00000000..757bbed2 --- /dev/null +++ b/test/parse-failure.test.ts @@ -0,0 +1,180 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { buildGraph } from '../src/engine/build.js'; +import { serializeGraph } from '../src/engine/serialize.js'; +import { parseSource } from '../src/engine/parse.js'; +import { PARSE_FAILURE_RECOVERY, parseFailedWarning } from '../src/engine/parse-failure.js'; +import { formatWarningLine, WARNING_CODES } from '../src/core-open/warnings.js'; +import { scanCommand } from '../src/reporting/commands/scan.js'; +import { cleanup, makeProject } from './helpers.js'; + +const PIN = '2020-01-01T00:00:00.000Z'; +const dirs: string[] = []; + +afterEach(() => { + while (dirs.length) cleanup(dirs.pop()!); +}); + +function project(files: Record): string { + const dir = makeProject(files); + dirs.push(dir); + return dir; +} + +/** Deliberate tree-sitter break: the grammar recovers with an ERROR node and no definitions. */ +const GAP = '@@@@\n'; + +function failureMessage(rel: string): string { + const stamped = parseFailedWarning(rel, 'ts'); + const suffix = ` [${WARNING_CODES.PARSE_FAILED}]`; + return stamped.endsWith(suffix) ? stamped.slice(0, -suffix.length) : stamped; +} + +describe('tree-sitter parse failure warning', () => { + it('names the file, the language, and a stable recovery hint', async () => { + const rel = 'src/a-gap.ts'; + const once = await parseSource(rel, 'ts', GAP); + const twice = await parseSource(rel, 'ts', GAP); + expect(once.warnings).toEqual(twice.warnings); + expect(once.warnings).toEqual([parseFailedWarning(rel, 'ts')]); + const message = failureMessage(rel); + expect(message).toContain(rel); + expect(message).toContain('(TypeScript)'); + expect(message).toContain('tree-sitter parse failed'); + expect(message).toContain(PARSE_FAILURE_RECOVERY); + expect(message).not.toContain('@@@@'); + expect(message).not.toMatch(/\bat \S+\(/); + expect(once.defs).toEqual([]); + }); + + it('does not warn for a file the grammar accepts', async () => { + const comment = await parseSource('src/note.ts', 'ts', '// just a comment\n'); + const expr = await parseSource('src/expr.ts', 'ts', '1 + 1;\n'); + const fn = await parseSource('src/kept.ts', 'ts', 'export function kept() { return 1; }\n'); + expect(comment.warnings ?? []).toEqual([]); + expect(expr.warnings ?? []).toEqual([]); + expect(fn.warnings ?? []).toEqual([]); + expect(fn.defs.map((d) => d.name)).toContain('kept'); + }); + + it('keeps recovered definitions and does not emit PARSE_FAILED for a partial tree', async () => { + const parsed = await parseSource('src/half.ts', 'ts', 'export function ok() { return 1; }\n@@@@\n'); + expect(parsed.defs.map((d) => d.name)).toContain('ok'); + expect(parsed.warnings ?? []).toEqual([]); + }); + + it('does not warn when a reused parser flags a valid file that has no definitions', async () => { + await parseSource('src/f.lua', 'lua', 'local function f()\n return 1\nend\n'); + const tiny = await parseSource('tiny.lua', 'lua', 'return 1\n'); + expect(tiny.defs).toEqual([]); + expect(tiny.warnings ?? []).toEqual([]); + }); + + it('keeps a recovered import and does not warn when a real construct remains', async () => { + const parsed = await parseSource('src/mixed.ts', 'ts', "import { kept } from './kept';\n@@@@\n"); + expect(parsed.defs).toEqual([]); + expect(parsed.imports.map((i) => i.source)).toEqual(['./kept']); + expect(parsed.warnings ?? []).toEqual([]); + }); + + it('replaces a parser crash with the same warning and still parses the next file', async () => { + const rel = 'scripts/case.sh'; + const crashed = await parseSource(rel, 'sh', 'case $x in\n a) echo hi;;\nesac\n'); + expect(crashed.defs).toEqual([]); + expect(crashed.warnings).toEqual([parseFailedWarning(rel, 'sh')]); + const text = crashed.warnings?.[0] ?? ''; + expect(text).toContain('scripts/case.sh'); + expect(text).toContain('(Shell)'); + expect(text).toContain(PARSE_FAILURE_RECOVERY); + expect(text).not.toContain('resolved is not a function'); + expect(text).not.toMatch(/\n/); + + const again = await parseSource(rel, 'sh', 'case $x in\n a) echo hi;;\nesac\n'); + expect(again.warnings).toEqual(crashed.warnings); + + const ok = await parseSource('scripts/ok.sh', 'sh', 'hello() {\n echo hi\n}\n'); + expect(ok.warnings ?? []).toEqual([]); + expect(ok.defs.map((d) => d.name)).toContain('hello'); + }); + + it('build warns in path order and keeps the graph deterministic', async () => { + const root = project({ + 'src/kept.ts': 'export function kept() { return 1; }\n', + 'src/note.ts': '// just a comment\n', + 'src/z-gap.ts': 'export function broken(\n', + 'src/a-gap.ts': GAP, + }); + const opts = { root, generatedAt: PIN, inline: true, noTsc: true, noIndex: true } as const; + const first = await buildGraph({ ...opts, noCache: true }); + const second = await buildGraph({ ...opts, noCache: true }); + const warmed = await buildGraph({ ...opts }); + + const messages = [failureMessage('src/a-gap.ts'), failureMessage('src/z-gap.ts')]; + expect(first.codedWarnings.map((warning) => warning.code)).toEqual([ + WARNING_CODES.PARSE_FAILED, + WARNING_CODES.PARSE_FAILED, + ]); + expect(first.codedWarnings.map((warning) => warning.message)).toEqual(messages); + expect(second.codedWarnings).toEqual(first.codedWarnings); + expect(warmed.codedWarnings).toEqual(first.codedWarnings); + expect(first.warnings).toEqual(messages.map((message) => `${message} [${WARNING_CODES.PARSE_FAILED}]`)); + expect(formatWarningLine(first.codedWarnings[0]!)).toBe(`warning [${WARNING_CODES.PARSE_FAILED}]: ${messages[0]}`); + + expect(serializeGraph(first.graph)).toBe(serializeGraph(second.graph)); + expect(serializeGraph(warmed.graph)).toBe(serializeGraph(first.graph)); + + const names = first.graph.nodes.map((node) => node.name); + expect(names).toContain('kept'); + expect(names).not.toContain('broken'); + const files = first.graph.nodes.filter((node) => node.kind === 'file').map((node) => node.file); + expect(files).toContain('src/a-gap.ts'); + expect(files).toContain('src/z-gap.ts'); + expect(files).toContain('src/kept.ts'); + expect(JSON.stringify(first.codedWarnings)).not.toContain('@@@@'); + expect(JSON.stringify(first.codedWarnings)).not.toContain('export function'); + }); +}); + +describe('vg scan surfaces parse failures', () => { + it('prints the same warning on stderr, twice identically', async () => { + const root = project({ + 'package.json': JSON.stringify({ name: 'parse-break-fixture', version: '1.0.0' }), + 'src/kept.ts': 'export function kept() { return 1; }\n', + 'src/a-gap.ts': GAP, + }); + const strip = (value: string) => value.replace(/\u001b\[[0-9;]*m/g, ''); + const run = async (): Promise => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + try { + await scanCommand.parseAsync([ + 'node', + 'scan', + root, + '--offline', + '--no-daemon', + '--quiet', + '--format', + 'json', + ]); + return strip(errorSpy.mock.calls.map((call) => call.map(String).join(' ')).join('\n')); + } finally { + errorSpy.mockRestore(); + logSpy.mockRestore(); + } + }; + + vi.stubEnv('VIBGRATE_DSN', ''); + vi.stubEnv('VIBGRATE_NO_KERNEL', '1'); + try { + const first = await run(); + const second = await run(); + const line = `warning [${WARNING_CODES.PARSE_FAILED}]: ${failureMessage('src/a-gap.ts')}`; + const lines = (text: string) => text.split('\n').filter((row) => row.includes(WARNING_CODES.PARSE_FAILED)); + expect(lines(first)).toEqual([line]); + expect(lines(second)).toEqual(lines(first)); + expect(first).not.toContain('@@@@'); + } finally { + vi.unstubAllEnvs(); + } + }, 120_000); +}); From 8281774898490c3cd19473e250c2966aa669a7a6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 10 Oct 2026 13:52:38 +0000 Subject: [PATCH 2/2] fix(engine): name the file when a parse fails A supported-language file the parser cannot read produces a stable VG_WARN_PARSE_FAILED warning with the path, the language, and the next step. The map continues without that file's symbols. The warning does not include a stack trace. vg build and vg scan both print it. Signed-off-by: Cursor Agent Co-authored-by: vibgrate-team --- ARCHITECTURE.md | 5 +- CHANGELOG.md | 2 +- DOCS.md | 2 +- src/engine/build.ts | 9 +- src/engine/languages.ts | 4 +- src/engine/parse-failure.ts | 28 ----- src/engine/parse-warning.ts | 34 ++++++ src/engine/parse-worker.ts | 6 +- src/engine/parse.ts | 66 ++++++----- src/engine/pool.ts | 6 +- src/reporting/commands/scan.ts | 17 +-- test/adversarial-fixtures.test.ts | 4 +- test/parse-failure-warning.test.ts | 81 +++++++++++++ test/parse-failure.test.ts | 180 ----------------------------- 14 files changed, 173 insertions(+), 271 deletions(-) delete mode 100644 src/engine/parse-failure.ts create mode 100644 src/engine/parse-warning.ts create mode 100644 test/parse-failure-warning.test.ts delete mode 100644 test/parse-failure.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6c8c34f5..475150fb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -133,10 +133,7 @@ asserts stable output. 1. **build** discovers files, parses them via the worker pool using bundled tree-sitter WASM grammars, resolves symbols/imports into typed relations, - runs analysis, and writes a stable `graph.json`. A registered-language file - whose parse throws, returns no tree, or recovers as only ERROR nodes with - no definitions emits `VG_WARN_PARSE_FAILED` (path, language, recovery - hint). `vg scan` prints that same warning when it builds the map. + runs analysis, and writes a stable `graph.json`. 2. **query** commands and the **MCP server** both load `graph.json` and answer questions against it — they do not re-parse the source. 3. **drift reporting** runs on top of `@vibgrate/core-open` and the resolved diff --git a/CHANGELOG.md b/CHANGELOG.md index 9712b117..e1aeda87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ backward compatible. ### Fixed -- **A supported-language file that fails tree-sitter parse warns with `VG_WARN_PARSE_FAILED`.** `vg build` and `vg scan` name the file path and the language, and tell you to fix the syntax or exclude the path with `--exclude`. The same file always produces the same warning. The map continues without that file's definitions. A parser crash is reported as that warning, not as a stack trace. (#289) +- **`vg build` and `vg scan` name a file when its parse fails.** A supported-language file the parser cannot read produces a warning with the file path, the language, and what to do next: correct the file or exclude the path. The map continues without that file's symbols. The same tree prints the same warning. The warning does not include a stack trace. (#289) - **SBOM component order no longer follows scan or filesystem order.** `vg sbom export` and CycloneDX/SPDX graph export sort a component by its diff --git a/DOCS.md b/DOCS.md index 7c37860b..92b47daa 100644 --- a/DOCS.md +++ b/DOCS.md @@ -5755,7 +5755,7 @@ A truncated or invalid lockfile stops the command. Warning codes cover condition | Code | Command | Meaning | | --- | --- | --- | -| `VG_WARN_PARSE_FAILED` | `vg build`, `vg scan` | A supported-language file failed tree-sitter parse. The warning names the file and the language, and tells you to fix the syntax or exclude the path with `--exclude`. The map continues without that file's definitions. The parser's own exception text is not the warning. | +| `VG_WARN_PARSE_FAILED` | `vg build`, `vg scan` | A supported-language source file failed to parse. The message names the file and the language. The map continues without that file's symbols. Correct the file or exclude the path with `--exclude`. | | `VG_WARN_BUILD_FILE_OVERSIZE` | `vg build` | A file exceeded the per-file size cap and was left out of the map. | | `VG_WARN_TSC_RESOLVER_SKIPPED` | `vg build` | The TypeScript resolver was skipped because the corpus exceeded its file cap. | | `VG_WARN_YAML_PARSE_FAILED` | `vg build` | YAML for an infrastructure file failed to parse. | diff --git a/src/engine/build.ts b/src/engine/build.ts index 19a365da..dca0bb65 100644 --- a/src/engine/build.ts +++ b/src/engine/build.ts @@ -51,7 +51,7 @@ import type { ResolveResult } from './resolve.js'; import { fileRolesFromParses } from './ast-roles.js'; import type { AstRoleHit } from '../core-open/scanners/architecture/ast-roles.js'; import { stampWarning, WARNING_CODES, type CodedWarning } from '../core-open/warnings.js'; -import { isParseFailedWarning } from './parse-failure.js'; +import { isParseFailureWarning } from './parse-warning.js'; import { assembleEngineWarnings } from './warning-codes.js'; export interface BuildOptions { @@ -309,11 +309,8 @@ export async function buildGraph(options: BuildOptions): Promise { // Never persist a failed parse: a transient parse-runtime crash (e.g. a // corrupted wasm heap) would otherwise poison the cache for that content // hash and every later build would reuse the empty parse instead of - // re-parsing the file. A syntax error that yielded no definitions is the - // same shape — re-parse it next time so a grammar fix is picked up, and - // so the warning is produced again from the file rather than from a - // cached empty table. - if (p.defs.length === 0 && p.warnings?.some((w) => isParseFailedWarning(w))) continue; + // re-parsing the file. + if (p.defs.length === 0 && p.warnings?.some((w) => isParseFailureWarning(w))) continue; const st = fileStats.find((f) => f.rel === p.rel); cache.set(p.rel, p, st ? { mtimeMs: st.mtimeMs, size: st.size } : undefined); } diff --git a/src/engine/languages.ts b/src/engine/languages.ts index 81a52ee3..20321b62 100644 --- a/src/engine/languages.ts +++ b/src/engine/languages.ts @@ -84,8 +84,8 @@ export const LANGUAGES: LanguageDef[] = [ { id: 'ex', label: 'Elixir', extensions: ['.ex', '.exs'], grammarFile: 'tree-sitter-elixir' }, // Known limitation: the bundled bash grammar's external scanner throws under // web-tree-sitter 0.25.10 on `case`/heredoc constructs. Such files degrade - // gracefully (per-file empty parse + a surfaced warning, never a build crash); - // functions in case/heredoc-free scripts extract normally. + // gracefully (per-file empty parse + a warning that names the file, never a + // build crash); functions in case/heredoc-free scripts extract normally. { id: 'sh', label: 'Shell', extensions: ['.sh', '.bash'], grammarFile: 'tree-sitter-bash' }, { id: 'zig', label: 'Zig', extensions: ['.zig'], grammarFile: 'tree-sitter-zig' }, { id: 'c', label: 'C', extensions: ['.c'], grammarFile: 'tree-sitter-c' }, diff --git a/src/engine/parse-failure.ts b/src/engine/parse-failure.ts deleted file mode 100644 index 17dcd2c2..00000000 --- a/src/engine/parse-failure.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { stampWarning, WARNING_CODES, warningPathLabel } from '../core-open/warnings.js'; -import { langById } from './languages.js'; - -/** - * Recovery sentence for {@link WARNING_CODES.PARSE_FAILED}. Stable: tests and - * CI assert this exact hint. It names no file contents and no exception text. - */ -export const PARSE_FAILURE_RECOVERY = 'Fix the syntax, or exclude the path with --exclude.'; - -/** - * Deterministic warning when a registered-language file fails tree-sitter - * parse. The path is repo-relative, the language is the registry label, and - * the hint is {@link PARSE_FAILURE_RECOVERY}. The parser's exception text is - * omitted — wasm failures often surface as a stack or a non-deterministic - * message, and that must not be the primary line. - */ -export function parseFailedWarning(rel: string, langId: string): string { - const language = langById(langId)?.label ?? langId; - return stampWarning( - WARNING_CODES.PARSE_FAILED, - `${warningPathLabel(rel)} (${language}): tree-sitter parse failed. The map continues without this file's definitions. ${PARSE_FAILURE_RECOVERY}`, - ); -} - -/** True for a stamped {@link WARNING_CODES.PARSE_FAILED} line, old or new. */ -export function isParseFailedWarning(warning: string): boolean { - return warning.endsWith(` [${WARNING_CODES.PARSE_FAILED}]`); -} diff --git a/src/engine/parse-warning.ts b/src/engine/parse-warning.ts new file mode 100644 index 00000000..837026c9 --- /dev/null +++ b/src/engine/parse-warning.ts @@ -0,0 +1,34 @@ +import { formatWarningLine, stampWarning, WARNING_CODES, warningPathLabel, type CodedWarning } from '../core-open/warnings.js'; +import { langById } from './languages.js'; + +/** + * Deterministic warning when a supported-language file cannot be parsed. + * + * The sentence is the whole operator-facing message: repo-relative path, + * language, and what to do next. It never includes a stack, a parser + * exception, or file contents — those can leak internals or source. + * Same path and language always produce the same string. + */ +export function parseFailureWarning(rel: string, langId: string): string { + const language = langById(langId)?.label ?? langId; + const file = warningPathLabel(rel); + return stampWarning( + WARNING_CODES.PARSE_FAILED, + `${file} (${language}): parse failed. The map continues without this file's symbols. Correct the file or exclude the path with --exclude.`, + ); +} + +/** True for a stamped parse-failure warning, including the older `parse failed:` prefix. */ +export function isParseFailureWarning(warning: string): boolean { + return warning.includes(`[${WARNING_CODES.PARSE_FAILED}]`) || warning.startsWith('parse failed:'); +} + +/** Stderr lines for parse failures, in the order `warnings` already has. */ +export function parseFailureWarningLines(warnings: readonly CodedWarning[]): string[] { + const lines: string[] = []; + for (const warning of warnings) { + if (warning.code !== WARNING_CODES.PARSE_FAILED) continue; + lines.push(formatWarningLine(warning)); + } + return lines; +} diff --git a/src/engine/parse-worker.ts b/src/engine/parse-worker.ts index c88a216c..abd1a0f0 100644 --- a/src/engine/parse-worker.ts +++ b/src/engine/parse-worker.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import { parseSource } from './parse.js'; import { setGrammarsOverride, resetParser } from './grammars.js'; import type { FileParse } from './types.js'; -import { parseFailedWarning } from './parse-failure.js'; +import { parseFailureWarning } from './parse-warning.js'; /** * tinypool worker entry. Receives a chunk of files, reads and parses each, and @@ -35,6 +35,8 @@ export default async function run(payload: ParsePayload): Promise { } catch { // A wasm-level parse crash can leave the language's reused parser // mid-state; drop it so the failure stays contained to this file. + // The exception text is intentionally dropped: it can be a wasm stack + // or a snippet of source. The warning names the file instead. resetParser(task.lang); out.push({ rel: task.rel, @@ -47,7 +49,7 @@ export default async function run(payload: ParsePayload): Promise { heritage: [], typeRefs: [], guards: [], - warnings: [parseFailedWarning(task.rel, task.lang)], + warnings: [parseFailureWarning(task.rel, task.lang)], }); } } diff --git a/src/engine/parse.ts b/src/engine/parse.ts index d81c41f2..57d3214e 100644 --- a/src/engine/parse.ts +++ b/src/engine/parse.ts @@ -1,6 +1,5 @@ import { Query, type Node, type Language } from 'web-tree-sitter'; -import { parserFor, loadLanguage, resetParser } from './grammars.js'; -import { parseFailedWarning } from './parse-failure.js'; +import { parserFor, loadLanguage } from './grammars.js'; import { langById } from './languages.js'; import { queriesFor, type DefRule } from './queries.js'; import { extractEmbeddedScript } from './sfc.js'; @@ -15,6 +14,7 @@ function effectsRegexEnabled(): boolean { return !(v === '0' || v === 'false'); } import { extractDutiesWithCandidates, fileBindings, type Bindings } from './duties.js'; +import { parseFailureWarning } from './parse-warning.js'; import type { FileParse, RawCall, RawDef, RawGuard, RawHeritage, RawImport, RawTypeRef } from './types.js'; /** @@ -288,17 +288,11 @@ export async function parseSource( const language = await loadLanguage(effLangId); const parser = await parserFor(def); - let tree; - try { - tree = parser.parse(text); - } catch { - // An external scanner can throw (wasm) and leave the reused parser - // mid-state. Drop it so the next file of this language starts clean. - // The warning is stable: the exception text is not part of the line. - resetParser(effLangId); - return markParseFailed(result); + const tree = parser.parse(text); + if (!tree) { + result.warnings = [parseFailureWarning(rel, langId)]; + return result; } - if (!tree) return markParseFailed(result); const root = tree.rootNode; // --- definitions --- @@ -477,34 +471,42 @@ export async function parseSource( const roles = extractAstRolesFromTree(rel, effLangId, language, root, text); if (roles) result.roles = roles; - // Decide before delete(): the node is invalid afterwards. A file whose root - // contains only ERROR nodes, and no definitions, used to disappear with no - // notice. A nested ERROR next to a real construct is not this case — some - // grammars set hasError on a reused parser for valid sources that simply - // declare nothing (Lua `return 1`), and warning there would be a false alarm. - const unrecovered = result.defs.length === 0 && unrecoveredTree(root); + // Tree-sitter returns a tree for broken syntax instead of throwing. When + // that tree is only ERROR nodes and nothing was extracted, the file was + // skipped. `hasError` alone is not that signal: some grammars set it on + // valid nodes, and a reused parser can set it on a later file that parsed + // cleanly on its own (Lua `return 1`). + if (treeIsOnlyErrors(root) && parseYieldedNothing(result)) { + result.warnings = [...(result.warnings ?? []), parseFailureWarning(rel, langId)]; + } + tree.delete(); - if (unrecovered) return markParseFailed(result); return result; } -/** True when the parse recovered nothing but ERROR nodes at the root. */ -function unrecoveredTree(root: Node): boolean { +function treeIsOnlyErrors(root: Node): boolean { if (!root.hasError) return false; - if (root.isError || root.type === 'ERROR') return true; - let real = 0; - let error = 0; - for (const kid of root.namedChildren) { - if (!kid) continue; - if (kid.isError || kid.type === 'ERROR') error += 1; - else real += 1; + if (root.type === 'ERROR') return true; + const count = root.namedChildCount; + if (count === 0) return false; + for (let i = 0; i < count; i++) { + const child = root.namedChild(i); + if (!child || child.type !== 'ERROR') return false; } - return error > 0 && real === 0; + return true; } -function markParseFailed(result: FileParse): FileParse { - result.warnings = [parseFailedWarning(result.rel, result.lang)]; - return result; +function parseYieldedNothing(result: FileParse): boolean { + return ( + result.defs.length === 0 && + result.calls.length === 0 && + result.imports.length === 0 && + result.heritage.length === 0 && + (result.typeRefs?.length ?? 0) === 0 && + (result.guards?.length ?? 0) === 0 && + (result.namespaces?.length ?? 0) === 0 && + (result.roles?.length ?? 0) === 0 + ); } /** diff --git a/src/engine/pool.ts b/src/engine/pool.ts index 653e90b4..8d98ecc8 100644 --- a/src/engine/pool.ts +++ b/src/engine/pool.ts @@ -7,7 +7,7 @@ import { setGrammarsOverride, resetParser } from './grammars.js'; import { checkMemoryBudget, envJobs, envWorkerHeapMb, ResourceLimitError } from './limits.js'; import type { DiscoveredFile } from './discover.js'; import type { FileParse } from './types.js'; -import { parseFailedWarning } from './parse-failure.js'; +import { parseFailureWarning } from './parse-warning.js'; import type { ParseTask } from './parse-worker.js'; /** @@ -81,8 +81,10 @@ async function parseInline(files: DiscoveredFile[], options: ParseOptions): Prom } catch { // A wasm-level parse crash can leave the language's reused parser // mid-state; drop it so the failure stays contained to this file. + // The exception text is intentionally dropped: it can be a wasm stack + // or a snippet of source. The warning names the file instead. resetParser(file.lang.id); - out.push(emptyParse(file, parseFailedWarning(file.rel, file.lang.id))); + out.push(emptyParse(file, parseFailureWarning(file.rel, file.lang.id))); } onProgress?.(out.length, files.length); if (out.length % MEM_CHECK_EVERY === 0) checkMemoryBudget('parse', memoryBudgetMb); diff --git a/src/reporting/commands/scan.ts b/src/reporting/commands/scan.ts index 22d532eb..7dbd4cb1 100644 --- a/src/reporting/commands/scan.ts +++ b/src/reporting/commands/scan.ts @@ -18,7 +18,6 @@ import { loadConfig, findConfigFile, } from '../../core-open/index.js'; -import { formatWarningLine, WARNING_CODES, type CodedWarning } from '../../core-open/warnings.js'; import { writeScanSummary } from '../scan-summary.js'; import { compareDriftBudget, evaluateConfigDriftBudget } from '../drift-budget-gate.js'; import { @@ -53,6 +52,7 @@ import { emitIngestIdLine, emitDriftScoreLine } from '../utils/ingest-id-output. import { formatUploadHttpFailure, uploadScanArtifact } from '../utils/upload.js'; import { redactForDisplay } from '../../core-open/utils/redact.js'; import { buildGraph } from '../../engine/build.js'; +import { parseFailureWarningLines } from '../../engine/parse-warning.js'; import { writeArtifacts, resolveGraphPath } from '../../engine/artifacts.js'; import { readHaileSidecar } from '../../engine/haile/sidecar.js'; import { isUsableHaileSymbol } from '../../engine/haile/format.js'; @@ -695,9 +695,6 @@ export const scanCommand = new Command('scan') // Retained by the postScan hook so the reachability query below can run // against the freshly built map without a second (memory-heavy) build. let builtGraph: VgGraph | null = null; - // Parse failures from the code-map walk. Printed after the scan report so - // the progress renderer cannot overwrite them, and before any gate exit. - let parseFailureWarnings: CodedWarning[] = []; // Outcome of the infrastructure pack run (`--iac`). `null` means it never // ran: no code map, or the map build failed before it. Only an `ok` run // puts `extended.security` on the artifact — the section is never @@ -712,7 +709,11 @@ export const scanCommand = new Command('scan') exclude: opts.exclude, onParseProgress: (done, total) => report(done, total, 'parsing'), }); - parseFailureWarnings = result.codedWarnings.filter((warning) => warning.code === WARNING_CODES.PARSE_FAILED); + // Same coded warning `vg build` prints. Stderr only, so scan JSON + // on stdout stays the artifact. One line per file, already sorted. + for (const line of parseFailureWarningLines(result.codedWarnings)) { + console.error(chalk.yellow(line)); + } builtGraph = result.graph; const written = writeArtifacts(result.graph, { root: rootDir }); if (written.architecturePolicyError) console.error(chalk.red(`\narchitecture policy: ${written.architecturePolicyError}`)); @@ -767,12 +768,6 @@ export const scanCommand = new Command('scan') const advanced = await loadAdvancedScanHook(); const artifact = await runCoreScan(rootDir, scanOpts, advanced); - // The map walk degrades per file. Surface the same stable line `vg build` - // prints so a scan does not omit a supported-language parse failure. - for (const warning of parseFailureWarnings) { - console.error(chalk.yellow(formatWarningLine(warning))); - } - // The scan just built a code map (its `postScan` step). Start the local // runtime if it is not up and hand it that map, so the very first `vg` in a // repo leaves behind a warm graph slot and a warming semantic index rather diff --git a/test/adversarial-fixtures.test.ts b/test/adversarial-fixtures.test.ts index 23adf9ca..5e457dd4 100644 --- a/test/adversarial-fixtures.test.ts +++ b/test/adversarial-fixtures.test.ts @@ -96,9 +96,9 @@ for (const lang of buildableLangs) { // (c) no parse failures. `BuildResult.warnings` is the engine's only // per-file failure signal (parse/query problems surface as - // VG_WARN_PARSE_FAILED); it must be empty. + // VG_WARN_PARSE_FAILED warnings); it must be empty. expect(first.warnings).toEqual([]); - expect(first.warnings.filter((w) => /query|VG_WARN_PARSE_FAILED|parse failed/i.test(w))).toEqual([]); + expect(first.warnings.filter((w) => /query|parse failed/i.test(w))).toEqual([]); // (d) determinism: rebuilding serializes byte-identically via the same // serializer the CLI writes graph.json with. For known re-parse-unstable diff --git a/test/parse-failure-warning.test.ts b/test/parse-failure-warning.test.ts new file mode 100644 index 00000000..03beb5c0 --- /dev/null +++ b/test/parse-failure-warning.test.ts @@ -0,0 +1,81 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { buildGraph } from '../src/engine/build.js'; +import { parseFailureWarning, parseFailureWarningLines } from '../src/engine/parse-warning.js'; +import { serializeGraph } from '../src/engine/serialize.js'; +import { cleanup, makeProject } from './helpers.js'; + +const PIN = '2020-01-01T00:00:00.000Z'; +const dirs: string[] = []; + +afterEach(() => { + while (dirs.length) cleanup(dirs.pop()!); +}); + +const SYNTAX_MSG = + "src/broken.ts (TypeScript): parse failed. The map continues without this file's symbols. Correct the file or exclude the path with --exclude."; +const SHELL_MSG = + "scripts/dispatch.sh (Shell): parse failed. The map continues without this file's symbols. Correct the file or exclude the path with --exclude."; + +function fixture(): string { + const root = makeProject({ + 'src/ok.ts': 'export function ok(): number { return 1; }\n', + 'src/value.ts': 'export const x = 1;\n', + 'src/partial.ts': 'export function kept(): number { return 1; }\n@@@\n', + 'src/broken.ts': 'export function broken(\n', + 'scripts/ok.sh': '#!/bin/sh\nhello() { echo hi; }\nhello\n', + // The bundled shell grammar throws on `case` (web-tree-sitter 0.25.x). + 'scripts/dispatch.sh': 'case "$1" in\n a) echo a ;;\nesac\n', + }); + dirs.push(root); + return root; +} + +function parseFailures(warnings: { code: string; message: string }[]): string[] { + return warnings.filter((warning) => warning.code === 'VG_WARN_PARSE_FAILED').map((warning) => warning.message); +} + +describe('parse failure warning', () => { + it('names the file and language, repeats exactly, and keeps recovered files', async () => { + const root = fixture(); + const opts = { root, generatedAt: PIN, inline: true, noCache: true }; + const first = await buildGraph(opts); + const second = await buildGraph(opts); + + expect(parseFailures(first.codedWarnings)).toEqual([SHELL_MSG, SYNTAX_MSG]); + expect(first.codedWarnings).toEqual(second.codedWarnings); + expect(first.warnings).toEqual(second.warnings); + expect(serializeGraph(first.graph)).toBe(serializeGraph(second.graph)); + + const text = parseFailures(first.codedWarnings).join('\n'); + expect(text).not.toMatch(/resolved is not a function/); + expect(text).not.toMatch(/\n\s+at /); + + const names = first.graph.nodes.map((node) => node.name); + expect(names).toContain('ok'); + expect(names).toContain('kept'); + expect(names).toContain('hello'); + expect(names).not.toContain('broken'); + }); + + it('still warns on a second build when the failed parse is not cached', async () => { + const root = fixture(); + const opts = { root, generatedAt: PIN, inline: true }; + const first = await buildGraph(opts); + const second = await buildGraph(opts); + expect(parseFailures(first.codedWarnings)).toEqual([SHELL_MSG, SYNTAX_MSG]); + expect(parseFailures(second.codedWarnings)).toEqual([SHELL_MSG, SYNTAX_MSG]); + }); + + it('keeps an escaping path to the file name and formats the scan line', () => { + const stored = parseFailureWarning('../secret.ts', 'ts'); + expect(stored).toContain('secret.ts (TypeScript)'); + expect(stored).not.toContain('..'); + expect(stored.endsWith(' [VG_WARN_PARSE_FAILED]')).toBe(true); + expect( + parseFailureWarningLines([ + { code: 'VG_WARN_PARSE_FAILED', message: SYNTAX_MSG }, + { code: 'VG_WARN_BUILD_FILE_OVERSIZE', message: 'src/big.ts: skipped' }, + ]), + ).toEqual([`warning [VG_WARN_PARSE_FAILED]: ${SYNTAX_MSG}`]); + }); +}); diff --git a/test/parse-failure.test.ts b/test/parse-failure.test.ts deleted file mode 100644 index 757bbed2..00000000 --- a/test/parse-failure.test.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { buildGraph } from '../src/engine/build.js'; -import { serializeGraph } from '../src/engine/serialize.js'; -import { parseSource } from '../src/engine/parse.js'; -import { PARSE_FAILURE_RECOVERY, parseFailedWarning } from '../src/engine/parse-failure.js'; -import { formatWarningLine, WARNING_CODES } from '../src/core-open/warnings.js'; -import { scanCommand } from '../src/reporting/commands/scan.js'; -import { cleanup, makeProject } from './helpers.js'; - -const PIN = '2020-01-01T00:00:00.000Z'; -const dirs: string[] = []; - -afterEach(() => { - while (dirs.length) cleanup(dirs.pop()!); -}); - -function project(files: Record): string { - const dir = makeProject(files); - dirs.push(dir); - return dir; -} - -/** Deliberate tree-sitter break: the grammar recovers with an ERROR node and no definitions. */ -const GAP = '@@@@\n'; - -function failureMessage(rel: string): string { - const stamped = parseFailedWarning(rel, 'ts'); - const suffix = ` [${WARNING_CODES.PARSE_FAILED}]`; - return stamped.endsWith(suffix) ? stamped.slice(0, -suffix.length) : stamped; -} - -describe('tree-sitter parse failure warning', () => { - it('names the file, the language, and a stable recovery hint', async () => { - const rel = 'src/a-gap.ts'; - const once = await parseSource(rel, 'ts', GAP); - const twice = await parseSource(rel, 'ts', GAP); - expect(once.warnings).toEqual(twice.warnings); - expect(once.warnings).toEqual([parseFailedWarning(rel, 'ts')]); - const message = failureMessage(rel); - expect(message).toContain(rel); - expect(message).toContain('(TypeScript)'); - expect(message).toContain('tree-sitter parse failed'); - expect(message).toContain(PARSE_FAILURE_RECOVERY); - expect(message).not.toContain('@@@@'); - expect(message).not.toMatch(/\bat \S+\(/); - expect(once.defs).toEqual([]); - }); - - it('does not warn for a file the grammar accepts', async () => { - const comment = await parseSource('src/note.ts', 'ts', '// just a comment\n'); - const expr = await parseSource('src/expr.ts', 'ts', '1 + 1;\n'); - const fn = await parseSource('src/kept.ts', 'ts', 'export function kept() { return 1; }\n'); - expect(comment.warnings ?? []).toEqual([]); - expect(expr.warnings ?? []).toEqual([]); - expect(fn.warnings ?? []).toEqual([]); - expect(fn.defs.map((d) => d.name)).toContain('kept'); - }); - - it('keeps recovered definitions and does not emit PARSE_FAILED for a partial tree', async () => { - const parsed = await parseSource('src/half.ts', 'ts', 'export function ok() { return 1; }\n@@@@\n'); - expect(parsed.defs.map((d) => d.name)).toContain('ok'); - expect(parsed.warnings ?? []).toEqual([]); - }); - - it('does not warn when a reused parser flags a valid file that has no definitions', async () => { - await parseSource('src/f.lua', 'lua', 'local function f()\n return 1\nend\n'); - const tiny = await parseSource('tiny.lua', 'lua', 'return 1\n'); - expect(tiny.defs).toEqual([]); - expect(tiny.warnings ?? []).toEqual([]); - }); - - it('keeps a recovered import and does not warn when a real construct remains', async () => { - const parsed = await parseSource('src/mixed.ts', 'ts', "import { kept } from './kept';\n@@@@\n"); - expect(parsed.defs).toEqual([]); - expect(parsed.imports.map((i) => i.source)).toEqual(['./kept']); - expect(parsed.warnings ?? []).toEqual([]); - }); - - it('replaces a parser crash with the same warning and still parses the next file', async () => { - const rel = 'scripts/case.sh'; - const crashed = await parseSource(rel, 'sh', 'case $x in\n a) echo hi;;\nesac\n'); - expect(crashed.defs).toEqual([]); - expect(crashed.warnings).toEqual([parseFailedWarning(rel, 'sh')]); - const text = crashed.warnings?.[0] ?? ''; - expect(text).toContain('scripts/case.sh'); - expect(text).toContain('(Shell)'); - expect(text).toContain(PARSE_FAILURE_RECOVERY); - expect(text).not.toContain('resolved is not a function'); - expect(text).not.toMatch(/\n/); - - const again = await parseSource(rel, 'sh', 'case $x in\n a) echo hi;;\nesac\n'); - expect(again.warnings).toEqual(crashed.warnings); - - const ok = await parseSource('scripts/ok.sh', 'sh', 'hello() {\n echo hi\n}\n'); - expect(ok.warnings ?? []).toEqual([]); - expect(ok.defs.map((d) => d.name)).toContain('hello'); - }); - - it('build warns in path order and keeps the graph deterministic', async () => { - const root = project({ - 'src/kept.ts': 'export function kept() { return 1; }\n', - 'src/note.ts': '// just a comment\n', - 'src/z-gap.ts': 'export function broken(\n', - 'src/a-gap.ts': GAP, - }); - const opts = { root, generatedAt: PIN, inline: true, noTsc: true, noIndex: true } as const; - const first = await buildGraph({ ...opts, noCache: true }); - const second = await buildGraph({ ...opts, noCache: true }); - const warmed = await buildGraph({ ...opts }); - - const messages = [failureMessage('src/a-gap.ts'), failureMessage('src/z-gap.ts')]; - expect(first.codedWarnings.map((warning) => warning.code)).toEqual([ - WARNING_CODES.PARSE_FAILED, - WARNING_CODES.PARSE_FAILED, - ]); - expect(first.codedWarnings.map((warning) => warning.message)).toEqual(messages); - expect(second.codedWarnings).toEqual(first.codedWarnings); - expect(warmed.codedWarnings).toEqual(first.codedWarnings); - expect(first.warnings).toEqual(messages.map((message) => `${message} [${WARNING_CODES.PARSE_FAILED}]`)); - expect(formatWarningLine(first.codedWarnings[0]!)).toBe(`warning [${WARNING_CODES.PARSE_FAILED}]: ${messages[0]}`); - - expect(serializeGraph(first.graph)).toBe(serializeGraph(second.graph)); - expect(serializeGraph(warmed.graph)).toBe(serializeGraph(first.graph)); - - const names = first.graph.nodes.map((node) => node.name); - expect(names).toContain('kept'); - expect(names).not.toContain('broken'); - const files = first.graph.nodes.filter((node) => node.kind === 'file').map((node) => node.file); - expect(files).toContain('src/a-gap.ts'); - expect(files).toContain('src/z-gap.ts'); - expect(files).toContain('src/kept.ts'); - expect(JSON.stringify(first.codedWarnings)).not.toContain('@@@@'); - expect(JSON.stringify(first.codedWarnings)).not.toContain('export function'); - }); -}); - -describe('vg scan surfaces parse failures', () => { - it('prints the same warning on stderr, twice identically', async () => { - const root = project({ - 'package.json': JSON.stringify({ name: 'parse-break-fixture', version: '1.0.0' }), - 'src/kept.ts': 'export function kept() { return 1; }\n', - 'src/a-gap.ts': GAP, - }); - const strip = (value: string) => value.replace(/\u001b\[[0-9;]*m/g, ''); - const run = async (): Promise => { - const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); - try { - await scanCommand.parseAsync([ - 'node', - 'scan', - root, - '--offline', - '--no-daemon', - '--quiet', - '--format', - 'json', - ]); - return strip(errorSpy.mock.calls.map((call) => call.map(String).join(' ')).join('\n')); - } finally { - errorSpy.mockRestore(); - logSpy.mockRestore(); - } - }; - - vi.stubEnv('VIBGRATE_DSN', ''); - vi.stubEnv('VIBGRATE_NO_KERNEL', '1'); - try { - const first = await run(); - const second = await run(); - const line = `warning [${WARNING_CODES.PARSE_FAILED}]: ${failureMessage('src/a-gap.ts')}`; - const lines = (text: string) => text.split('\n').filter((row) => row.includes(WARNING_CODES.PARSE_FAILED)); - expect(lines(first)).toEqual([line]); - expect(lines(second)).toEqual(lines(first)); - expect(first).not.toContain('@@@@'); - } finally { - vi.unstubAllEnvs(); - } - }, 120_000); -});