parser: HTML and CSS front end on tree-sitter, with specification and torture suites

.gitattributes

@@ -29,3 +29,7 @@
 *.jpg       binary
 *.pdf       binary
 *.sqlite    binary
+
+# A torture fixture whose CRLF line endings ARE the trap it sets (a BOM and CRLF in one
+# page); normalising it would remove the case the suite asserts.
+parser/src/test-data/web/torture/quirks.html -text

bin/axiomcode

@@ -393,7 +393,7 @@ case "$cmd" in
     run_suite(){ local s="$1"; shift; echo "══ $s"; bash "$ROOT/graph/test/$s/run-tests.sh" "$@" || rc=1; }
     case "$which" in
       java|typescript|python|javascript|csharp) run_suite "$which" "$@";;
-      parser) for t in java-extractor-tests typescript-tests python-tests javascript-tests csharp-tests gradle-tests services-tests discovery-tests; do
+      parser) for t in java-extractor-tests typescript-tests python-tests javascript-tests csharp-tests gradle-tests services-tests web-tests web-torture discovery-tests; do
                 [ -f "$ROOT/parser/src/test/$t.ts" ] || continue; echo "══ parser: $t"; (cd "$ROOT/parser" && npx tsx "src/test/$t.ts") || rc=1; done;;
       all) for l in java typescript python javascript csharp; do run_suite "$l" "$@"; done; "$0" test parser || rc=1;;
       *) die "test: unknown suite $which (java | typescript | python | javascript | csharp | parser | all)";;

package.json

@@ -55,10 +55,13 @@
     "LICENSE.md"
   ],
   "dependencies": {
+    "entities": "^6.0.0",
     "sax": "^1.4.4",
     "tree-sitter": "^0.21.1",
     "tree-sitter-c-sharp": "0.23.1",
+    "tree-sitter-css": "0.23.0",
     "tree-sitter-groovy": "^0.1.2",
+    "tree-sitter-html": "0.23.2",
     "tree-sitter-java": "0.23.4",
     "tree-sitter-python": "^0.21.0",
     "typescript": "^5.3.3",

parser/README.md

@@ -89,6 +89,23 @@ same tables and be compared.
 | **YAML** | Beta | 2 | yaml | Configuration entries with anchor and alias tracking, multi document support. |
 | **META-INF/services** | Stable | 2 | custom | Provider-configuration files: the service each file configures, taken from its name, and every implementation class it names, with the file and line. |
 | **Gradle** | Beta | 8 | tree-sitter-groovy | Scripts and their role in the build, blocks, declarations, dependency coordinates split into group/artifact/version, version catalogs, value references with resolution, comments, and parse gaps. Groovy and Kotlin DSL. |
+| **HTML** | Beta | 9 | tree-sitter-html | Documents with their doctype and template dialects, the element tree as written (nothing a browser would imply) with XPath-like paths, attributes by kind, class tokens, every URL a page names (scripts, stylesheets, links, images, forms, frames, htmx requests) classified and resolved to a file where one exists, `<script>` elements with their type and inline body range (inside inline `<svg>` too), the calls written in event-handler attributes, `javascript:` URLs and framework event bindings, template expressions (Vue, Angular, Alpine, Jinja, Handlebars, ERB and the others: the directive, its argument and modifiers, the expression text, the names it calls and reads, the variables a loop declares), and parse gaps. `.html`, `.htm`, `.xhtml`. Inline `<style>` elements and `style` attributes are extracted as CSS. |
+| **CSS** | Beta | 8 | tree-sitter-css | Stylesheets (files and `<style>` elements), rules and at-rules as a tree including CSS nesting, selectors with specificity, simple selectors (type, class, id, attribute, pseudo-class, pseudo-element, nesting) with combinators, declarations, the names values refer to (custom properties, URLs, imports, keyframes, layers, containers, font families), comments and parse gaps. `.css` only; a preprocessor dialect is recorded as a gap, not parsed. |
+
+HTML and CSS are one front end, extracted structurally the way the configuration formats are:
+a page's `<link rel="stylesheet">` and `<script src>` are resolved to the files they name, a
+`class` attribute's tokens and a stylesheet's class selectors are rows that join by name, and a
+handler attribute's calls carry their callee names for the engine to resolve against the scripts
+the page loads. Nothing is rendered: a template dialect (Jinja, Thymeleaf, Handlebars, Razor, ERB,
+PHP, Angular, Vue, Alpine, htmx) is detected from its markers and recorded on the document, so a
+consumer knows a URL holding `{{` is a template's and not a path. Each template construct is also
+a row of its own: `v-for="item in items"` is a LOOP that declares `item` and reads `items`,
+`@click="save(id)"` is an EVENT_HANDLER whose call `save` is a handler-call row like an `onclick`'s,
+`:class="{ on: open }"` is a BINDING of `class` that reads `open`, `{{ user.name }}` an INTERPOLATION,
+and a Jinja `{% for %}` or an Angular `*ngIf` the same with their own dialect. The expression inside is
+read by the TypeScript syntax layer where the dialect's expression language is JavaScript's, and a
+value that does not parse is a gap on the row, not a guess. A `<base href>` is honoured when URLs
+are resolved, as a browser would.
 
 Python and Java are the two languages with full semantic resolution. TypeScript and C# resolve what
 one file decides and leave the cross-file call graph to the engine, as Java does for type references. The configuration formats are
@@ -172,7 +189,7 @@ detection  ->  parsing  ->  extraction  ->  models  ->  resolution  ->  export
 | Layer | Directory | Responsibility |
 |---|---|---|
 | Detection | `language-detectors/` | Identify which languages and build systems a project uses. |
-| Parsing | `parsers/<lang>/` | Produce a syntax tree. tree-sitter for Java, Python, C# and Gradle; the TypeScript compiler's syntax layer for TypeScript and JavaScript; sax for XML; the `yaml` package for YAML; a hand written scanner for Properties and for META-INF/services. |
+| Parsing | `parsers/<lang>/` | Produce a syntax tree. tree-sitter for Java, Python, C#, Gradle, HTML and CSS; the TypeScript compiler's syntax layer for TypeScript and JavaScript; sax for XML; the `yaml` package for YAML; a hand written scanner for Properties and for META-INF/services. |
 | Extraction | `parsers/<lang>/extractors/` | Walk the tree and emit rows. One extractor per relation family, implementing `BaseExtractor`. |
 | Models | `analysis-types/<lang>/` | One class per relation. Builder pattern, content addressed key, CSV serialisation. |
 | Resolution | `parsers/<lang>/**/*-resolution-linker.ts` | Fill in cross entity foreign keys, first within a file and then across the project. |
@@ -251,6 +268,24 @@ Every gate therefore uses something not written for this purpose.
 | A runtime tracer | Which TypeScript declaration a call actually reaches | A source rewrite that records, as a project's own test suite runs, the declaration entered at each call site. The compiler says what it resolved; this says what ran, and the two disagree in ways that matter. |
 | Gradle `projects` | The build's project graph | Gradle is the implementation that decides which projects a settings file creates. On its first real run it found a directory this parser was reporting as a project and Gradle was not. |
 
+The web front end's oracle is `tools/web-corpus-bench`, a development-only bench that runs the
+front end and reference parsers (parse5 and htmlparser2 for HTML; css-tree, postcss and
+`@bramus/specificity` for CSS) over downloaded corpora (html5lib, web-platform-tests, the csstree
+and postcss fixtures, several real applications, live pages and CDN stylesheets) and reports every
+measure on which they differ; its last baseline is committed next to it. tree-sitter-html and
+tree-sitter-css build the trees, and both grammars are narrower than the languages: the HTML grammar knows void elements,
+raw text and implicit end tags but does not run the browser's tree construction (nothing is
+implied, a stray end tag is an error), and the CSS grammar rejects some valid shapes — an unquoted
+`url(../x)`, an attribute selector's `i` flag, a `@container` name, a range media query, `&` after
+a compound — each of which the front end reads back from the source text and marks as recovered
+rather than as a gap. What this repository adds (URL classification and resolution, specificity,
+the names a value refers to, the calls in a handler, positions through an inline `<style>`, the
+recoveries) is checked by format checks written from the HTML, CSS Syntax, Selectors and URL
+specifications, each paired with the wrong implementation it rules out, by an enum-emission
+audit over the fixture, and by the torture suite below. A browser-side oracle (a DOM built by a
+browser engine, `getComputedStyle` for the cascade) would be the next independent check, and is
+not there.
+
 Call graph quality is measured at three increasing strictnesses, because each answers a question the
 previous cannot. **Discovery** asks whether a call site was found at all, against the compiled
 bytecode, which lists every call whether or not the branch runs. **Target correctness** asks whether
@@ -349,11 +384,14 @@ Both call the same core in `src/extract.ts`.
 
 Tab separated files, one per relation. Java relations are named `all-*.csv`; every other language
 is prefixed by name: `all-python-*.csv`, `all-typescript-*.csv`, `all-javascript-*.csv`,
-`all-csharp-*.csv`, `all-gradle-*.csv`, `all-xml-*.csv` and so on. The `skipped-*-files.csv` files
+`all-csharp-*.csv`, `all-gradle-*.csv`, `all-xml-*.csv`, `all-html-*.csv`, `all-css-*.csv` and so on. The `skipped-*-files.csv` files
 record every file that was not analysed and why, and a file the extractor loses part way leaves a row
 saying so, so a consumer can distinguish an empty result from an unanalysed one.
 
-`--per-language` writes `outputDir/<lang>/` instead of one flat folder. `--library` marks the tree as
+`--per-language` writes `outputDir/<lang>/` instead of one flat folder. The HTML and CSS tables are
+copied into every folder among `java/`, `javascript/` and `typescript/` that exists, since a page is
+read by whichever of those engines the repository runs; their frozen schema is
+`src/schema/web/schema.json` and the matching declarations `src/schema/web/decls_base_web.dl`. `--library` marks the tree as
 a dependency being staged rather than the project under analysis, so a build output directory its
 `package.json` ships from is walked as its source.
 
@@ -376,6 +414,8 @@ npx tsx src/test/javascript-tests.ts
 npx tsx src/test/csharp-tests.ts
 npx tsx src/test/gradle-tests.ts
 npx tsx src/test/services-tests.ts
+npx tsx src/test/web-tests.ts
+npx tsx src/test/web-torture.ts
 npx tsx src/test/discovery-tests.ts
 npx tsx src/test/java-gates/source-walk.ts
 ```
@@ -388,6 +428,19 @@ and torture scripts — dense files where each line is a known trap for a hand-r
 the language's answer asserted by line. `--corpus <dir>` adds sweeps over a real corpus that are
 development-only: absent from the plain run and failing, not passing, when the corpus is missing.
 
+The web suite is split the same way. `web-tests.ts` is the specification: format checks with the
+wrong implementation each rules out, a fixture tree with a golden for drift, key uniqueness,
+foreign-key integrity, schema arity and domains, determinism and the enum-emission audit.
+`web-torture.ts` is the adversary: it runs the analyzer over `src/test-data/web/torture`, a project
+written to hold every trap a static reader of a web project meets (a `<base href>`, links with query
+strings and fragments, `../` escaping the root, uppercase `REL`, declarative shadow roots, inline
+SVG with its own `<style>` and `<script>`, XHTML, quirks-mode markup, Jinja and Vue templates,
+escaped and non-ASCII class names, CSS nesting, layers, container queries, keyframes, `@import`
+chains, minified and 90 KB sheets) and asserts, join by join, that the IR carries what the
+documented joins need. Every check carries its verdict when it fails: a DEFECT (the parser could have
+read it) fails the run; a GAP (the IR cannot yet express it) and a LIMIT (nothing static can know
+it) are reported, so the list of what the front end does not do is explicit and kept current.
+
 What ships is what Python and TypeScript ship: fixtures, in-repo gates, committed expectations. The
 oracle harness that computes an expectation, and the `bless` command that writes it, live in a
 separate repository so that a suite cannot authorise its own expectations. One consequence is real

parser/package.json

@@ -16,10 +16,13 @@
     "node": ">=18.0.0"
   },
   "dependencies": {
+    "entities": "6.0.1",
     "sax": "^1.4.4",
     "tree-sitter": "^0.21.1",
     "tree-sitter-c-sharp": "0.23.1",
+    "tree-sitter-css": "0.23.0",
     "tree-sitter-groovy": "^0.1.2",
+    "tree-sitter-html": "0.23.2",
     "tree-sitter-java": "0.23.4",
     "tree-sitter-python": "^0.21.0",
     "typescript": "^5.3.3",

parser/src/analysis-types/css/CssComment.ts

@@ -0,0 +1,49 @@
+import { WEB_COMMENT_TEXT_LIMIT } from '@/constants/web-constants';
+
+import { WebRow, boundedText, num } from '../web/web-row';
+
+/** A `/* … *\/` comment, with its text, for parity with the other front ends. */
+export class CssComment extends WebRow {
+  static readonly COLUMNS = [
+    'text', 'startLine', 'startColumn', 'endLine', 'endColumn', 'stylesheetLinkHash',
+    'serviceVersionLinkHash', 'cssCommentUniqueHash',
+  ] as const;
+
+  readonly relation = 'css_comment';
+  readonly columns = CssComment.COLUMNS;
+
+  readonly text: string;
+  readonly startLine: number;
+  readonly startColumn: number;
+  readonly endLine: number;
+  readonly endColumn: number;
+  readonly stylesheetLinkHash: string;
+  readonly serviceVersionLinkHash: string;
+
+  constructor(props: {
+    text: string; startLine: number; startColumn: number; endLine: number; endColumn: number;
+    stylesheetLinkHash: string; serviceVersionLinkHash: string;
+  }) {
+    super();
+    this.text = props.text;
+    this.startLine = props.startLine;
+    this.startColumn = props.startColumn;
+    this.endLine = props.endLine;
+    this.endColumn = props.endColumn;
+    this.stylesheetLinkHash = props.stylesheetLinkHash;
+    this.serviceVersionLinkHash = props.serviceVersionLinkHash;
+    this.generateHash();
+  }
+
+  /** **PK** `CSS_COMMENT_md5(stylesheetLinkHash ‖ startLine ‖ startColumn)`. */
+  generateHash(): void {
+    this.hash = WebRow.key('CSS_COMMENT', this.stylesheetLinkHash, this.startLine, this.startColumn);
+  }
+
+  protected values(): string[] {
+    return [
+      boundedText(this.text, WEB_COMMENT_TEXT_LIMIT), num(this.startLine), num(this.startColumn),
+      num(this.endLine), num(this.endColumn), this.stylesheetLinkHash, this.serviceVersionLinkHash, this.hash,
+    ];
+  }
+}

parser/src/analysis-types/css/CssDeclaration.ts

@@ -0,0 +1,77 @@
+import { WEB_TEXT_LIMIT } from '@/constants/web-constants';
+
+import { WebRow, bool, boundedText, num, text } from '../web/web-row';
+
+/**
+ * One `property: value` pair.
+ *
+ * It belongs to exactly one of two owners, and the two are separate columns rather
+ * than one polymorphic link: `ruleLinkHash` for a declaration inside a rule block,
+ * `htmlAttributeLinkHash` for one written in a `style="…"` attribute. A polymorphic
+ * key defeats a referential-integrity gate — "the hash exists in one of two tables"
+ * is not integrity — so each column points at one relation and exactly one is set.
+ * `property` is kept as written: a custom property is case-sensitive and a vendor
+ * prefix is information.
+ */
+export class CssDeclaration extends WebRow {
+  static readonly COLUMNS = [
+    'property', 'valueText', 'isImportant', 'isCustomProperty', 'vendorPrefix', 'position',
+    'startLine', 'startColumn', 'endLine', 'endColumn', 'ruleLinkHash', 'htmlAttributeLinkHash',
+    'stylesheetLinkHash', 'serviceVersionLinkHash', 'cssDeclarationUniqueHash',
+  ] as const;
+
+  readonly relation = 'css_declaration';
+  readonly columns = CssDeclaration.COLUMNS;
+
+  readonly property: string;
+  readonly valueText: string;
+  readonly isImportant: boolean;
+  readonly isCustomProperty: boolean;
+  readonly vendorPrefix: string;
+  readonly position: number;
+  readonly startLine: number;
+  readonly startColumn: number;
+  readonly endLine: number;
+  readonly endColumn: number;
+  readonly ruleLinkHash: string;
+  readonly htmlAttributeLinkHash: string;
+  readonly stylesheetLinkHash: string;
+  readonly serviceVersionLinkHash: string;
+
+  constructor(props: {
+    property: string; valueText: string; isImportant: boolean; isCustomProperty: boolean; vendorPrefix: string;
+    position: number; startLine: number; startColumn: number; endLine: number; endColumn: number;
+    ruleLinkHash: string; htmlAttributeLinkHash: string; stylesheetLinkHash: string; serviceVersionLinkHash: string;
+  }) {
+    super();
+    this.property = props.property;
+    this.valueText = props.valueText;
+    this.isImportant = props.isImportant;
+    this.isCustomProperty = props.isCustomProperty;
+    this.vendorPrefix = props.vendorPrefix;
+    this.position = props.position;
+    this.startLine = props.startLine;
+    this.startColumn = props.startColumn;
+    this.endLine = props.endLine;
+    this.endColumn = props.endColumn;
+    this.ruleLinkHash = props.ruleLinkHash;
+    this.htmlAttributeLinkHash = props.htmlAttributeLinkHash;
+    this.stylesheetLinkHash = props.stylesheetLinkHash;
+    this.serviceVersionLinkHash = props.serviceVersionLinkHash;
+    this.generateHash();
+  }
+
+  /** **PK** `CSS_DECLARATION_md5(ruleLinkHash ‖ htmlAttributeLinkHash ‖ position)` — one owner is empty. */
+  generateHash(): void {
+    this.hash = WebRow.key('CSS_DECLARATION', this.ruleLinkHash, this.htmlAttributeLinkHash, this.position);
+  }
+
+  protected values(): string[] {
+    return [
+      text(this.property), boundedText(this.valueText, WEB_TEXT_LIMIT), bool(this.isImportant),
+      bool(this.isCustomProperty), text(this.vendorPrefix), num(this.position), num(this.startLine),
+      num(this.startColumn), num(this.endLine), num(this.endColumn), this.ruleLinkHash,
+      this.htmlAttributeLinkHash, this.stylesheetLinkHash, this.serviceVersionLinkHash, this.hash,
+    ];
+  }
+}

parser/src/analysis-types/css/CssParseGap.ts

@@ -0,0 +1,55 @@
+import { WEB_COMMENT_TEXT_LIMIT } from '@/constants/web-constants';
+import { CssParseGapKind } from '@/enums/css/CssParseGapKind';
+
+import { WebRow, boundedText, num } from '../web/web-row';
+
+/** One region of a stylesheet whose rows are missing or suspect, recorded as data. */
+export class CssParseGap extends WebRow {
+  static readonly COLUMNS = [
+    'gapKind', 'detail', 'startLine', 'startColumn', 'endLine', 'endColumn', 'relatedRuleLinkHash',
+    'stylesheetLinkHash', 'serviceVersionLinkHash', 'cssParseGapUniqueHash',
+  ] as const;
+
+  readonly relation = 'css_parse_gap';
+  readonly columns = CssParseGap.COLUMNS;
+
+  readonly gapKind: CssParseGapKind;
+  readonly detail: string;
+  readonly startLine: number;
+  readonly startColumn: number;
+  readonly endLine: number;
+  readonly endColumn: number;
+  readonly relatedRuleLinkHash: string;
+  readonly stylesheetLinkHash: string;
+  readonly serviceVersionLinkHash: string;
+
+  constructor(props: {
+    gapKind: CssParseGapKind; detail: string; startLine: number; startColumn: number; endLine: number;
+    endColumn: number; relatedRuleLinkHash: string; stylesheetLinkHash: string; serviceVersionLinkHash: string;
+  }) {
+    super();
+    this.gapKind = props.gapKind;
+    this.detail = props.detail;
+    this.startLine = props.startLine;
+    this.startColumn = props.startColumn;
+    this.endLine = props.endLine;
+    this.endColumn = props.endColumn;
+    this.relatedRuleLinkHash = props.relatedRuleLinkHash;
+    this.stylesheetLinkHash = props.stylesheetLinkHash;
+    this.serviceVersionLinkHash = props.serviceVersionLinkHash;
+    this.generateHash();
+  }
+
+  /** **PK** `CSS_PARSE_GAP_md5(stylesheetLinkHash ‖ gapKind ‖ startLine ‖ startColumn ‖ detail)`. */
+  generateHash(): void {
+    this.hash = WebRow.key('CSS_PARSE_GAP', this.stylesheetLinkHash, this.gapKind, this.startLine, this.startColumn, this.detail);
+  }
+
+  protected values(): string[] {
+    return [
+      this.gapKind, boundedText(this.detail, WEB_COMMENT_TEXT_LIMIT), num(this.startLine), num(this.startColumn),
+      num(this.endLine), num(this.endColumn), this.relatedRuleLinkHash, this.stylesheetLinkHash,
+      this.serviceVersionLinkHash, this.hash,
+    ];
+  }
+}