Prepare Highlight 1.0 with stable contracts and runtime validation

.changeset/highlight-stable-contract.md

@@ -0,0 +1,5 @@
+---
+"@tanstack/highlight": major
+---
+
+Release the stable 1.0 highlighting and adapter contracts. Correct JavaScript and TypeScript keyword property names while preserving switch defaults, accessors, strings, and comments. Validate installed package exports and bounded malformed input across supported Node runtimes, and document migration and compatibility guarantees.

.github/workflows/ci.yml

@@ -25,3 +25,37 @@ jobs:
         uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3
       - name: Verify
         run: pnpm run verify
+
+      - name: Pack runtime artifact
+        run: mkdir -p runtime-package && npm pack --ignore-scripts --pack-destination runtime-package
+      - name: Upload runtime artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: highlight-runtime
+          path: runtime-package/*.tgz
+
+  runtime:
+    needs: verify
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        node: [18, 20, 22, 24, 26]
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
+        with:
+          node-version: ${{ matrix.node }}
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: highlight-runtime
+          path: runtime-package
+      - name: Install packed package and verify runtime
+        run: |
+          mkdir consumer
+          cd consumer
+          npm init -y
+          npm install --ignore-scripts --no-audit --no-fund ../runtime-package/*.tgz
+          node ../scripts/test-runtime.mjs

README.md

@@ -31,6 +31,7 @@ This is not an editor parser or a TextMate engine. It is a deliberately small do
 ## Documentation
 
 - [Overview](docs/overview.md)
+- [Migrating to v1 and compatibility policy](docs/guides/migrating-to-v1.md)
 - [Installation](docs/installation.md)
 - [Quick Start](docs/quick-start.md)
 - [Comparison](docs/comparison.md)
@@ -214,10 +215,12 @@ Local browser bundles, minified with esbuild and compressed independently. KB us
 | Registration | Minified | Gzip | Brotli |
 | --- | ---: | ---: | ---: |
 | Core, no languages | 3.84 KB | 1.82 KB | 1.66 KB |
-| Core + TSX | 9.46 KB | 4.03 KB | 3.66 KB |
-| Octane MDX + TypeScript | 13.21 KB | 5.37 KB | 4.90 KB |
-| Nine-language docs set | 15.39 KB | 5.97 KB | 5.44 KB |
-| All 30 languages | 30.08 KB | 10.61 KB | 9.56 KB |
+| Core + TSX | 10.11 KB | 4.26 KB | 3.90 KB |
+| Octane MDX + TypeScript | 13.86 KB | 5.59 KB | 5.17 KB |
+| Nine-language docs set | 16.04 KB | 6.20 KB | 5.66 KB |
+| All 30 languages | 30.72 KB | 10.77 KB | 9.79 KB |
+
+The following comparison was measured before the 1.0 property-context correction. Re-run the comparison commands below for current timings.
 
 On 80 real JavaScript/TypeScript/JSX/TSX TanStack docs fixtures repeated across 5,040 blocks, using the median of three runs after warmup:
 

docs/comparison.md

@@ -73,3 +73,5 @@ Choose another tool when you need:
 - Semantic tokens from a language service
 
 Run `pnpm run compare:sugar-high` and `pnpm run compare:shiki` to reproduce the repository's comparisons.
+
+These recorded comparison timings precede the 1.0 property-context correction. Run the comparison scripts for current measurements; see the performance guide for current bundle sizes.

docs/config.json

@@ -12,6 +12,7 @@
         { "label": "Overview", "to": "overview" },
         { "label": "Installation", "to": "installation" },
         { "label": "Quick Start", "to": "quick-start" },
+        { "label": "Migrating to v1", "to": "guides/migrating-to-v1" },
         { "label": "Comparison", "to": "comparison" }
       ]
     },

docs/guides/migrating-to-v1.md

@@ -0,0 +1,27 @@
+---
+title: Migrating to v1
+---
+
+# Migrating to v1
+
+Install `@tanstack/highlight@^1.0.0`. Existing 0.x version ranges do not select 1.x automatically. Update your lockfile and use the same Highlight version and language registrations for server and client rendering.
+
+The 1.0 release preserves the 0.1 public entry points and synchronous APIs. No import or option changes are required. JavaScript and TypeScript property names that are also keywords now receive property styling. This can change token snapshots and colors without changing the source text.
+
+If upgrading from an older 0.0 release, review the changelog for parser corrections, additional languages and adapters. Check rendered code blocks, copy text, line annotations and your theme selectors. `highlight()` preserves input; the view helpers `renderCodeBlockData()` and `renderCodeFence()` intentionally trim trailing whitespace before rendering and copying.
+
+## Compatibility policy
+
+Within 1.x, exported entry points, documented argument and return shapes, the meaning of existing `th-*` classes, and documented HTML/HAST wrapper structure remain compatible. Removal or incompatible changes require a major release. This includes language aliases and fallback behavior, UTF-16 decoration offsets, and the structured Markdown adapters.
+
+Parser corrections are patch releases. They may change which existing semantic class is assigned to a token, or split and join adjacent token spans. Exact token boundaries, complete HTML snapshots and syntax coloring are not frozen. Source preservation and escaped output remain required.
+
+New languages, themes and optional settings are minor releases. `HighlightTokenClass` is a closed public union, so adding or removing a token class requires a major release to preserve exhaustive TypeScript consumers. Changing an existing class's meaning also requires a major release.
+
+Custom language definitions and theme selectors are trusted application configuration. The renderer escapes code and decoration attributes; it does not sanitize arbitrary CSS supplied to theme helpers. Synchronous highlighting is intended for documentation-sized blocks. Applications accepting unbounded input should enforce their own size limit or isolate work off the UI thread.
+
+## Runtime support
+
+The published ESM package supports Node.js 18 and newer and modern browsers. Build tooling can require a newer Node version than the published runtime package. CI imports every installed public entry point and runs representative and bounded malformed inputs on Node 18, 20, 22, 24 and 26. Browser and SSR users should share a registry and avoid re-highlighting server-rendered blocks during hydration.
+
+Highlight is a documentation highlighter, not an incremental editor parser or a compiler grammar. These remain outside the 1.0 contract.

docs/guides/performance.md

@@ -13,10 +13,10 @@ CI measures selective browser bundles and highlighting performance on real docum
 | Profile | Registered languages | Current gzip | CI budget |
 | --- | --- | ---: | ---: |
 | Core | None | 1.82 KB | 2.0 KB |
-| TSX | TSX | 4.03 KB | 4.1 KB |
-| Octane | TypeScript plus Octane MDX adapter | 5.37 KB | 5.5 KB |
-| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 5.97 KB | 6.1 KB |
-| All | All 30 definitions | 10.61 KB | 10.8 KB |
+| TSX | TSX | 4.26 KB | 4.35 KB |
+| Octane | TypeScript plus Octane MDX adapter | 5.59 KB | 5.7 KB |
+| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.20 KB | 6.3 KB |
+| All | All 30 definitions | 10.77 KB | 10.9 KB |
 
 KB uses 1,000 bytes. Core helpers imported from the root tree-shake to the same engine size. The standalone theme helper is 695 gzip bytes.
 
@@ -75,3 +75,5 @@ Context-aware fixes are welcome when they solve common docs code. A change shoul
 - Generated HTML size when output structure changes
 
 The correct response to a crossed budget is to inspect the behavior and architecture. Budgets can move when a measured quality improvement justifies the bytes, but the tradeoff must be explicit.
+
+The 1.0 property-context correction adds roughly 230 gzip bytes to the TSX profile without changing core. Local Node 26 gzip results differ slightly from CI compression: the CI docs profile is 6,217 bytes and all languages is 10,838 bytes. Their budgets are 6,300 and 10,900 bytes respectively, retaining a small explicit margin.

docs/test-strategy.md

@@ -24,10 +24,10 @@ The suite protects the package's actual product boundary: valid code commonly pu
 | Profile | Languages | Gzip budget |
 | --- | --- | ---: |
 | Core | None | 2.0 KB |
-| TSX | TSX | 4.1 KB |
-| Octane | TypeScript plus Octane MDX adapter | 5.5 KB |
-| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.1 KB |
-| All | All 30 definitions | 10.8 KB |
+| TSX | TSX | 4.35 KB |
+| Octane | TypeScript plus Octane MDX adapter | 5.7 KB |
+| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.3 KB |
+| All | All 30 definitions | 10.9 KB |
 
 The selective profiles are the primary product metric. The all-language profile protects the convenience entry from unbounded growth. Bundle graphs reject unexpected language or theme code. Package tests repeat isolation checks through public exports after building.
 
@@ -39,4 +39,4 @@ The selective profiles are the primary product metric. The all-language profile
 
 ## Deliberate Omissions
 
-The package does not maintain compiler conformance suites, malformed-input fuzzing, ReDoS corpora, editor state tests, or exact parity snapshots against another highlighter. Those would optimize for a broader parser product than this library intends to become.
+Installed-package checks include bounded malformed strings, repeated delimiters, Unicode and escaping under a process timeout. They are a regression guard, not a proof of arbitrary-input complexity. The package does not maintain compiler conformance suites, unbounded fuzzing, a comprehensive ReDoS corpus, editor state tests, or exact parity snapshots against another highlighter. Those would optimize for a broader parser product than this library intends to become.

package.json

@@ -118,8 +118,9 @@
     "skills:check-version": "node scripts/sync-skill-version.mjs --check",
     "skills:sync-version": "node scripts/sync-skill-version.mjs",
     "typecheck": "tsc --noEmit",
-    "verify": "pnpm run typecheck && pnpm run build && pnpm run test:docs && pnpm run test:skills && pnpm run lint:package && pnpm run test:package && pnpm run test && pnpm run size && pnpm run bench",
-    "version": "pnpm run skills:sync-version"
+    "verify": "pnpm run typecheck && pnpm run build && pnpm run test:docs && pnpm run test:skills && pnpm run lint:package && pnpm run test:package && pnpm run test:packed && pnpm run test && pnpm run size && pnpm run bench",
+    "version": "pnpm run skills:sync-version",
+    "test:packed": "node scripts/test-packed.mjs"
   },
   "devDependencies": {
     "@changesets/cli": "2.31.1",

scripts/measure-size.mjs

@@ -29,7 +29,7 @@ const profiles = {
       globalThis.highlighter = createHighlighter({ languages: [tsx] })
     `,
     languages: ['tsx'],
-    limits: { minified: 9_800, gzip: 4_100, brotli: 3_750 },
+    limits: { minified: 10_200, gzip: 4_350, brotli: 4_000 },
   },
   tsxBarrel: {
     source: `
@@ -38,7 +38,7 @@ const profiles = {
       globalThis.highlighter = createHighlighter({ languages: [tsx] })
     `,
     languages: ['tsx'],
-    limits: { minified: 9_800, gzip: 4_100, brotli: 3_750 },
+    limits: { minified: 10_200, gzip: 4_350, brotli: 4_000 },
   },
   octane: {
     source: `
@@ -50,7 +50,7 @@ const profiles = {
       globalThis.octaneHighlight = createOctaneMdxHighlight({ highlighter })
     `,
     languages: ['ts'],
-    limits: { minified: 13_500, gzip: 5_500, brotli: 5_000 },
+    limits: { minified: 14_000, gzip: 5_700, brotli: 5_250 },
   },
   docs: {
     source: `
@@ -69,7 +69,7 @@ const profiles = {
       })
     `,
     languages: ['css', 'html', 'js', 'json', 'jsx', 'markdown', 'shell', 'ts', 'tsx'],
-    limits: { minified: 16_000, gzip: 6_100, brotli: 5_550 },
+    limits: { minified: 16_100, gzip: 6_300, brotli: 5_700 },
   },
   php: {
     source: `
@@ -104,7 +104,7 @@ const profiles = {
       globalThis.highlighter = defaultHighlighter
     `,
     languages: 'all',
-    limits: { minified: 30_500, gzip: 10_800, brotli: 9_800 },
+    limits: { minified: 30_800, gzip: 10_900, brotli: 9_800 },
   },
   reactAdapter: {
     source: `export * from './src/react.ts'`,

scripts/test-packed.mjs

@@ -0,0 +1,15 @@
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { execFileSync } from 'node:child_process'
+
+const consumer = mkdtempSync(join(tmpdir(), 'highlight-consumer-'))
+const env = { ...process.env, npm_config_cache: join(consumer, 'cache') }
+try {
+  const packed = JSON.parse(execFileSync('npm', ['pack', '--ignore-scripts', '--json', '--pack-destination', consumer], { encoding: 'utf8', env }))[0]
+  writeFileSync(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' }))
+  execFileSync('npm', ['install', '--ignore-scripts', '--offline', '--no-audit', '--no-fund', '--package-lock=false', join(consumer, packed.filename)], { cwd: consumer, stdio: 'inherit', env })
+  execFileSync(process.execPath, [resolve('scripts/test-runtime.mjs')], { cwd: consumer, stdio: 'inherit', env })
+} finally {
+  rmSync(consumer, { recursive: true, force: true })
+}

scripts/test-runtime.mjs

@@ -0,0 +1,56 @@
+import assert from 'node:assert/strict'
+import { readFileSync, readdirSync } from 'node:fs'
+import { createRequire } from 'node:module'
+import { spawnSync } from 'node:child_process'
+
+// Run from an installed package consumer, not the source checkout.
+const require = createRequire(`${process.cwd()}/consumer.cjs`)
+const entry = require.resolve('@tanstack/highlight')
+const root = new URL('../', `file://${entry}`)
+const pkg = JSON.parse(readFileSync(new URL('package.json', root), 'utf8'))
+const entries = Object.keys(pkg.exports).flatMap(key => {
+  if (!key.includes('*')) return [key === '.' ? pkg.name : pkg.name + key.slice(1)]
+  const directory = key.slice(2, key.indexOf('*'))
+  return readdirSync(new URL(`dist/${directory}`, root))
+    .filter(file => file.endsWith('.js'))
+    .map(file => `${pkg.name}/${directory}${file.slice(0, -3)}`)
+})
+// Imports resolve relative to this generated consumer module.
+const moduleSource = `
+import assert from 'node:assert/strict';
+const modules = await Promise.all(${JSON.stringify(entries)}.map(entry => import(entry)));
+const { highlight, listLanguages } = modules[0];
+const inputs = [
+  '<script>alert("x")</script> & \\"',
+  '/*' + '*'.repeat(20000),
+  '\\x60' + '\\x24{'.repeat(100),
+  '<'.repeat(10000),
+  '('.repeat(10000),
+  'a'.repeat(20000) + ':',
+  '😀\\r\\nconst x = { default: true }\\n',
+];
+for (const lang of listLanguages()) {
+  for (const code of inputs) {
+    const result = highlight(code, { lang });
+    assert.equal(result.tokens.map(token => token.value).join(''), code);
+    assert.ok(!result.html.includes('<script>'));
+    const recovered = result.html.replace(/<[^>]*>/g, '')
+      .replace(/&#39;/g, "'").replace(/&quot;/g, '\"')
+      .replace(/&gt;/g, '>').replace(/&lt;/g, '<').replace(/&amp;/g, '&');
+    assert.equal(recovered, code, lang + ' escaped HTML must preserve source');
+  }
+}
+const {createHighlighter} = await import('@tanstack/highlight/core');
+const {ts} = await import('@tanstack/highlight/languages/ts');
+const highlighter = createHighlighter({languages:[ts]});
+const result = highlighter.highlight('const value = 1', {lang:'ts'});
+assert.ok(result.html.includes('th-keyword'));
+assert.equal(highlighter.highlight('<unsafe>', {lang:'unknown'}).html.includes('<unsafe>'), false);
+console.log('Installed '+${JSON.stringify(pkg.name + '@' + pkg.version)}+': '+modules.length+' exports, all-language bounded hostile-input checks, and selective quick start passed on '+process.version);
+`
+const result = spawnSync(process.execPath, ['--input-type=module', '-e', moduleSource], {
+  cwd: process.cwd(), encoding: 'utf8', timeout: 20000,
+})
+assert.ifError(result.error)
+assert.equal(result.status, 0, result.stderr)
+console.log(result.stdout.trim())

src/internal/script.ts

@@ -5,6 +5,12 @@ const keywords =
   'abstract|as|asserts|async|await|break|case|catch|class|const|continue|debugger|declare|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|infer|instanceof|interface|is|keyof|let|module|namespace|new|of|override|package|private|protected|public|readonly|return|satisfies|set|static|super|switch|this|throw|try|type|typeof|using|var|while|with|yield'
 
 const semanticPatterns = [
+  {
+    className: 'property' as const,
+    regex: /(?<!\.)\.([A-Za-z_$][\w$]*)/g,
+    group: 1,
+  },
+
   { className: 'function' as const, regex: /@[A-Za-z_$][\w$]*/g },
   {
     className: 'keyword' as const,
@@ -34,11 +40,6 @@ const semanticPatterns = [
     regex:
       /\b(?:Array|Record|Promise|Readonly|Set|Map|WeakMap|WeakSet|string|number|boolean|bigint|symbol|object|unknown|never|void|any|[A-Z][A-Za-z0-9_$]*)\b/g,
   },
-  {
-    className: 'property' as const,
-    regex: /(?:\.|\?\.)([A-Za-z_$][\w$]*)/g,
-    group: 1,
-  },
 ] as const
 
 export function collectScriptRanges(
@@ -68,6 +69,7 @@ function collectScriptInitialRanges(
   limit = code.length,
   tagBody = false,
 ) {
+  const objectBraces: Array<boolean> = []
   const expressions: Array<number> = []
   let jsxDepth = 0
 
@@ -147,6 +149,31 @@ function collectScriptInitialRanges(
       }
     }
 
+    if (character === '{') {
+      const before = previousCodeIndex(code, index, ranges)
+      let wordStart = before
+      while (wordStart >= 0 && /[\w$]/.test(code[wordStart])) wordStart--
+      const word = code.slice(wordStart + 1, before + 1)
+      objectBraces.push(
+        '=(:,['.includes(code[before] || '\0') ||
+          /^(?:return|yield|const|let|var)$/.test(word),
+      )
+    } else if (character === '}') {
+      objectBraces.pop()
+    } else if (objectBraces.at(-1) && /[A-Za-z_$]/.test(character)) {
+      const before = previousCodeIndex(code, index, ranges)
+      if ('{,;'.includes(code[before] || '\0')) {
+        let end = index + 1
+        while (end < limit && /[\w$]/.test(code[end])) end++
+        const after = end + (code.slice(end).match(/^(?:\s|\/\*[\s\S]*?\*\/|\/\/[^\n]*(?:\n|$))*/)?.[0].length || 0)
+        if (code[after] === ':') {
+          ranges.push({ start: index, end, className: 'property' })
+          index = end
+          continue
+        }
+      }
+    }
+
     if (!jsx) {
       index++
       continue
@@ -433,3 +460,24 @@ function findLineEnd(code: string, start: number) {
   const end = code.indexOf('\n', start)
   return end < 0 ? code.length : end
 }
+
+// Comments do not change the syntactic position of a property or opening brace.
+function previousCodeIndex(code: string, index: number, ranges: Array<TokenRange>) {
+  let before = index - 1
+  let rangeIndex = ranges.length - 1
+  while (before >= 0) {
+    if (/\s/.test(code[before])) {
+      before--
+      continue
+    }
+    const range = ranges[rangeIndex]
+    if (range && range.end > before) {
+      rangeIndex--
+      if (range.className === 'comment' && range.start <= before) {
+        before = range.start - 1
+        continue
+      }
+    } else break
+  }
+  return before
+}