diff --git a/.changeset/more-languages.md b/.changeset/more-languages.md new file mode 100644 index 0000000..33a3968 --- /dev/null +++ b/.changeset/more-languages.md @@ -0,0 +1,5 @@ +--- +"@tanstack/highlight": minor +--- + +Add isolated Java, Kotlin, Rust, Ruby, C#, Dart, Lua, and Perl language definitions. Core and existing selective bundles are unchanged. diff --git a/README.md b/README.md index d75ba3a..36cc574 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Available themes: Aurora X, Dracula, GitHub Dark, GitHub Light, Gruvbox Dark, Gr ## Languages -`apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `python`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. +`apache`, `cmake`, `cpp`, `csharp`, `css`, `dart`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `java`, `js`, `json`, `jsx`, `kotlin`, `lua`, `markdown`, `mermaid`, `nginx`, `perl`, `php`, `plaintext`, `python`, `ruby`, `rust`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. Each language is available from `@tanstack/highlight/languages/`. The aggregate `@tanstack/highlight/languages` entry can tree-shake, while direct subpaths make isolation explicit. Importing only core helpers from the root entry also removes unused language registrations in a compatible bundler. @@ -218,7 +218,7 @@ Local browser bundles, minified with esbuild and compressed independently. KB us | Core + TSX | 10.11 KB | 4.26 KB | 3.90 KB | | Octane MDX + TypeScript | 13.86 KB | 5.59 KB | 5.17 KB | | Nine-language docs set | 16.04 KB | 6.20 KB | 5.66 KB | -| All 30 languages | 30.72 KB | 10.77 KB | 9.79 KB | +| All 38 languages | 49.99 KB | 16.82 KB | 15.19 KB | The following comparison was measured before the 1.0 property-context correction. Re-run the comparison commands below for current timings. diff --git a/docs/guides/performance.md b/docs/guides/performance.md index 11df7d7..36a659d 100644 --- a/docs/guides/performance.md +++ b/docs/guides/performance.md @@ -8,7 +8,7 @@ CI measures selective browser bundles and highlighting performance on real docum ## Bundle profiles -`pnpm run size` builds seventeen browser profiles with esbuild and measures minified, gzip, and Brotli bytes independently. It also checks that helper, adapter, and selective language imports retain only the requested modules. +`pnpm run size` builds twenty-five browser profiles with esbuild and measures minified, gzip, and Brotli bytes independently. It also checks that helper, adapter, and selective language imports retain only the requested modules. | Profile | Registered languages | Current gzip | CI budget | | --- | --- | ---: | ---: | @@ -16,7 +16,7 @@ CI measures selective browser bundles and highlighting performance on real docum | TSX | TSX | 4.26 KB | 4.35 KB | | Octane | TypeScript plus Octane MDX adapter | 5.59 KB | 5.7 KB | | Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.20 KB | 6.3 KB | -| All | All 30 definitions | 10.77 KB | 10.9 KB | +| All | All 38 definitions | 16.82 KB | 17.1 KB | KB uses 1,000 bytes. Core helpers imported from the root tree-shake to the same engine size. The standalone theme helper is 695 gzip bytes. @@ -26,7 +26,7 @@ Selective profiles are the primary metric. The all-language profile exists to pr The committed corpus contains 334 real code fences sampled from TanStack documentation, with up to twenty samples per normalized language. -`pnpm run bench` measures tokenization, HTML, Markdown, HAST, line numbers, long decorated blocks, and dedicated C++, CMake, and PHP samples. Each profile reports the median of three samples after two warmup passes, with a 1.2 second CI budget. The main highlighting profile processes at least 10,000 blocks. +`pnpm run bench` measures tokenization, HTML, Markdown, HAST, line numbers, long decorated blocks, and dedicated C#, C++, CMake, Dart, Java, Kotlin, Lua, Perl, PHP, Ruby, and Rust samples. Each profile reports the median of three samples after two warmup passes, with a 1.2 second CI budget. The main highlighting profile processes at least 10,000 blocks. A local before-and-after review used the same minified bundle settings, fixtures, and benchmark harness on macOS arm64 with Node 24.15.0: @@ -76,4 +76,4 @@ Context-aware fixes are welcome when they solve common docs code. A change shoul The correct response to a crossed budget is to inspect the behavior and architecture. Budgets can move when a measured quality improvement justifies the bytes, but the tradeoff must be explicit. -The 1.0 property-context correction adds roughly 230 gzip bytes to the TSX profile without changing core. Local Node 26 gzip results differ slightly from CI compression: the CI docs profile is 6,217 bytes and all languages is 10,838 bytes. Their budgets are 6,300 and 10,900 bytes respectively, retaining a small explicit margin. +The 1.0 property-context correction adds roughly 230 gzip bytes to the TSX profile without changing core. Local Node 26 gzip results differ slightly from CI compression: the CI docs profile is 6,217 bytes against a 6,300-byte budget, and the all-language budget keeps a comparable allowance above its local measurement. diff --git a/docs/installation.md b/docs/installation.md index 498e3cd..27b44f5 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ export const highlighter = createHighlighter({ }) ``` -The root entry is useful for prototypes, server-only scripts, or sites where the roughly 11 KB gzip all-language build is acceptable: +The root entry is useful for prototypes, server-only scripts, or sites where the roughly 17 KB gzip all-language build is acceptable: ```ts import { highlight } from '@tanstack/highlight' diff --git a/docs/language-support.md b/docs/language-support.md index 319e952..62bbcb2 100644 --- a/docs/language-support.md +++ b/docs/language-support.md @@ -13,7 +13,9 @@ Every language is an isolated definition imported from `@tanstack/highlight/lang | Apache | `apache` | - | Directives, tags, comments | | CMake | `cmake` | - | Bracket strings/comments, nested variables, generator expressions | | C++ | `cpp` | `c++`, `cc`, `cxx`, `hpp`, `hxx` | Raw and prefixed strings, character literals, preprocessor directives, digit separators | +| C# | `csharp` | `c#`, `cs` | Verbatim, interpolated, and raw strings with nested quotes, character literals, preprocessor directives, line-leading and parameter attributes, contextual and LINQ keywords | | CSS | `css` | - | Strings and comments protect inner syntax | +| Dart | `dart` | - | Interpolated strings with nested quotes and braces, triple-quoted and raw strings, nested block comments, annotations, named-argument labels, contextual keywords | | Diff | `diff` | `patch` | Metadata, inserted, and deleted lines | | Dockerfile | `dockerfile` | `docker` | Common directives, variables, commands | | EJS | `ejs` | - | HTML plus optional JavaScript delegation | @@ -21,15 +23,21 @@ Every language is an isolated definition imported from `@tanstack/highlight/lang | Go | `go` | `golang` | Raw strings, runes, comments, declarations | | HTML | `html` | `htm`, `xml`, `angular-html` | Optional JavaScript/TypeScript and CSS delegation | | HTTP | `http` | - | Methods, headers, protocol, paths | +| Java | `java` | - | Text blocks, character literals, annotations, contextual keywords, digit separators and hex floats | | JavaScript | `js` | `javascript`, `mjs`, `cjs`, `js-vue` | Templates, interpolation, regex literals | | JSON | `json` | `jsonc`, `json5` | Properties, comments, strings, literals | | JSX | `jsx` | - | JavaScript plus contextual JSX tags | +| Kotlin | `kotlin` | `kt`, `kts` | String templates with nested quotes and braces, raw strings, nested block comments, annotations vs labels, backticked names, contextual soft keywords and accessors | +| Lua | `lua` | - | Level-matched long-bracket strings and comments, string escapes, Luau backtick strings, shebang lines, paren-less calls, LuaJIT number suffixes | | Markdown | `markdown` | `md` | Optional fenced-language delegation | | Mermaid | `mermaid` | - | Common diagram declarations and arrows | | Nginx | `nginx` | - | Directives, variables, URLs, comments | +| Perl | `perl` | `pl` | Sigil and special variables, quote-like operators with nested delimiters, regex vs division, heredocs, POD blocks | | PHP | `php` | - | PHP tags, attributes, quoted strings, heredoc/nowdoc, optional HTML delegation | | Plaintext | `plaintext` | `text`, `txt`, `-->` | Escaping only | | Python | `python` | `py` | Triple strings, prefixes, decorators, comments | +| Ruby | `ruby` | `rb` | Interpolated strings with nested quotes, percent literals, heredocs, regex vs division, symbols and hash keys, block comments | +| Rust | `rust` | `rs` | Nested block comments, raw strings with hash counts, byte and C strings, lifetimes vs character literals, attributes, macros | | Scheme | `scheme` | `scm`, `racket` | Comments, strings, forms, literals | | Shell | `shell` | `bash`, `sh`, `zsh`, `cmd`, `console` | Heredocs, parameter expansion, comment boundaries | | SQL | `sql` | - | Strings, comments, common SQL clauses | diff --git a/docs/reference/default-entry.md b/docs/reference/default-entry.md index 7fbdfcb..3827e28 100644 --- a/docs/reference/default-entry.md +++ b/docs/reference/default-entry.md @@ -43,7 +43,7 @@ Returns the canonical registered language name for a name or alias. Names are tr function listLanguages(): Array ``` -Returns the 30 canonical language names registered in `defaultHighlighter`. +Returns the 38 canonical language names registered in `defaultHighlighter`. ### `tokenize` diff --git a/docs/reference/languages.md b/docs/reference/languages.md index b2f5132..3cf6f09 100644 --- a/docs/reference/languages.md +++ b/docs/reference/languages.md @@ -24,7 +24,9 @@ const highlighter = createHighlighter({ | `apache` | `@tanstack/highlight/languages/apache` | None | | `cmake` | `@tanstack/highlight/languages/cmake` | None | | `cpp` | `@tanstack/highlight/languages/cpp` | `c++`, `cc`, `cxx`, `hpp`, `hxx` | +| `csharp` | `@tanstack/highlight/languages/csharp` | `c#`, `cs` | | `css` | `@tanstack/highlight/languages/css` | None | +| `dart` | `@tanstack/highlight/languages/dart` | None | | `diff` | `@tanstack/highlight/languages/diff` | `patch` | | `dockerfile` | `@tanstack/highlight/languages/dockerfile` | `docker` | | `ejs` | `@tanstack/highlight/languages/ejs` | None | @@ -32,15 +34,21 @@ const highlighter = createHighlighter({ | `go` | `@tanstack/highlight/languages/go` | `golang` | | `html` | `@tanstack/highlight/languages/html` | `htm`, `xml`, `angular-html` | | `http` | `@tanstack/highlight/languages/http` | None | +| `java` | `@tanstack/highlight/languages/java` | None | | `js` | `@tanstack/highlight/languages/js` | `javascript`, `mjs`, `cjs`, `js-vue` | | `json` | `@tanstack/highlight/languages/json` | `jsonc`, `json5` | | `jsx` | `@tanstack/highlight/languages/jsx` | None | +| `kotlin` | `@tanstack/highlight/languages/kotlin` | `kt`, `kts` | +| `lua` | `@tanstack/highlight/languages/lua` | None | | `markdown` | `@tanstack/highlight/languages/markdown` | `md` | | `mermaid` | `@tanstack/highlight/languages/mermaid` | None | | `nginx` | `@tanstack/highlight/languages/nginx` | None | +| `perl` | `@tanstack/highlight/languages/perl` | `pl` | | `php` | `@tanstack/highlight/languages/php` | None | | `plaintext` | `@tanstack/highlight/languages/plaintext` | `text`, `txt`, `-->` | | `python` | `@tanstack/highlight/languages/python` | `py` | +| `ruby` | `@tanstack/highlight/languages/ruby` | `rb` | +| `rust` | `@tanstack/highlight/languages/rust` | `rs` | | `scheme` | `@tanstack/highlight/languages/scheme` | `scm`, `racket` | | `shell` | `@tanstack/highlight/languages/shell` | `bash`, `sh`, `zsh`, `cmd`, `console` | | `sql` | `@tanstack/highlight/languages/sql` | None | @@ -52,6 +60,6 @@ const highlighter = createHighlighter({ | `vue` | `@tanstack/highlight/languages/vue` | None | | `yaml` | `@tanstack/highlight/languages/yaml` | `yml` | -`@tanstack/highlight/languages` re-exports `apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `python`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. The barrel is convenient but individual subpaths make bundle intent explicit. +`@tanstack/highlight/languages` re-exports `apache`, `cmake`, `cpp`, `csharp`, `css`, `dart`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `java`, `js`, `json`, `jsx`, `kotlin`, `lua`, `markdown`, `mermaid`, `nginx`, `perl`, `php`, `plaintext`, `python`, `ruby`, `rust`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. The barrel is convenient but individual subpaths make bundle intent explicit. See the [language support matrix](../language-support) for the context-aware behavior and current scope of each registration. diff --git a/docs/test-strategy.md b/docs/test-strategy.md index 06d4e1d..4237776 100644 --- a/docs/test-strategy.md +++ b/docs/test-strategy.md @@ -19,7 +19,7 @@ The suite protects the package's actual product boundary: valid code commonly pu ## Size Profiles -`pnpm run size` checks seventeen independent browser profiles, including root helpers, language barrel imports, adapters, and themes. Each has minified, gzip, and Brotli budgets. The main highlighter profiles are: +`pnpm run size` checks twenty-five independent browser profiles, including root helpers, language barrel imports, adapters, and themes. Each has minified, gzip, and Brotli budgets. The main highlighter profiles are: | Profile | Languages | Gzip budget | | --- | --- | ---: | @@ -27,13 +27,13 @@ The suite protects the package's actual product boundary: valid code commonly pu | TSX | TSX | 4.35 KB | | Octane | TypeScript plus Octane MDX adapter | 5.7 KB | | Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.3 KB | -| All | All 30 definitions | 10.9 KB | +| All | All 38 definitions | 17.1 KB | The selective profiles are the primary product metric. The all-language profile protects the convenience entry from unbounded growth. Bundle graphs reject unexpected language or theme code. Package tests repeat isolation checks through public exports after building. ## Throughput -`pnpm run bench` measures highlighting, tokenization, Markdown, HAST, line numbers, long numbered blocks, long decorated blocks, and dedicated C++, CMake, and PHP samples. Timings use the median of three samples after two warmup passes. Each profile has a 1.2 second CI budget; the main highlighting profile processes at least 10,000 blocks. +`pnpm run bench` measures highlighting, tokenization, Markdown, HAST, line numbers, long numbered blocks, long decorated blocks, and dedicated C#, C++, CMake, Dart, Java, Kotlin, Lua, Perl, PHP, Ruby, and Rust samples. Timings use the median of three samples after two warmup passes. Each profile has a 1.2 second CI budget; the main highlighting profile processes at least 10,000 blocks. `pnpm run compare:sugar-high` compares the overlapping JS/TS/JSX/TSX use case. `pnpm run compare:shiki` compares all supported fixtures. These are directional measurements, not claims of equivalent grammar depth. diff --git a/scripts/bench.mjs b/scripts/bench.mjs index 12d8830..f8f7b97 100644 --- a/scripts/bench.mjs +++ b/scripts/bench.mjs @@ -68,6 +68,54 @@ try { observe: (result) => result.html.length, targetBlocks: 10_000, }, + csharp: { + fixtures: [{ rawLang: 'csharp', code: '#nullable enable\n[Obsolete]\nvar path = @"C:\\\\temp""x""";\nvar text = $"Hi {user.Name} {(ok ? "yes" : "no")}";\nvar query = from item in items where item.Count > 1_000 select item;\nchar c = \'\\n\'; /* done */' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + dart: { + fixtures: [{ rawLang: 'dart', code: '@immutable\nclass Point {\n final greeting = \'Hi ${user.name} ${map[\'key\']}\';\n final raw = r\'C:\\path\\$name\';\n /* outer /* nested */ comment */\n void show() => print(\'\'\'multi\nline\'\'\', count: 0x1F);\n}' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + java: { + fixtures: [{ rawLang: 'java', code: '@Override\npublic record Point(int x, int y) {}\nString text = """\n Hello "world"\n """;\nchar c = \'\\u0041\';\nvar mask = 0x7fff_ffff; double d = 0x1.8p1; // done' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + kotlin: { + fixtures: [{ rawLang: 'kotlin', code: '@file:JvmName("Main")\nval greeting = "Hi ${user.name.let { "[$it]" }} $count"\nval raw = """C:\\path ${\'$\'}x"""\n/* outer /* nested */ comment */\nfun `my test`() = loop@ for (i in 0..10) break@loop' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + lua: { + fixtures: [{ rawLang: 'lua', code: '#!/usr/bin/env lua\nlocal text = [==[ long ]] string ]==]\n--[[ block\ncomment ]]\nlocal n = 0x10ULL + 1.5e3\nrequire "module"\nprint("tab\\t" .. text)' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + perl: { + fixtures: [{ rawLang: 'perl', code: 'my %h = (key => $ARGV[0], list => [@_]);\nmy $q = qw{a {b} c};\n$x =~ s/foo/bar/g; my $y = $a / $b;\nprint <<"END";\nHello $name\nEND\n=pod\nDocs\n=cut' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + ruby: { + fixtures: [{ rawLang: 'ruby', code: 'greeting = "Hi #{user.name} #{h["key"]}"\nwords = %w[alpha beta]\ntext = <<~SQL\n SELECT * FROM t\nSQL\nmatch = value =~ /a+b/; ratio = a / b\nopts = { key: :value, "other" => 1 }\n=begin\nnote\n=end' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, + rust: { + fixtures: [{ rawLang: 'rust', code: '#[derive(Debug)]\nstruct Wrapper<\'a> { text: &\'a str }\n/* outer /* nested */ comment */\nlet raw = r#"quote "inside""#;\nlet bytes = b"data"; let c = c"text"; let ch = \'x\';\nprintln!("{}", vec![1u8, 0xff]);' }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 10_000, + }, tokenize: { fixtures, run: (fixture) => tokenize(fixture.code, { lang: fixture.rawLang }), diff --git a/scripts/language-utils.mjs b/scripts/language-utils.mjs index ca6fe86..39dad9c 100644 --- a/scripts/language-utils.mjs +++ b/scripts/language-utils.mjs @@ -5,7 +5,9 @@ export const supportedLanguages = [ 'apache', 'cmake', 'cpp', + 'csharp', 'css', + 'dart', 'diff', 'dockerfile', 'ejs', @@ -13,15 +15,21 @@ export const supportedLanguages = [ 'go', 'html', 'http', + 'java', 'js', 'json', 'jsx', + 'kotlin', + 'lua', 'markdown', 'mermaid', 'nginx', + 'perl', 'php', 'plaintext', 'python', + 'ruby', + 'rust', 'scheme', 'shell', 'sql', @@ -40,6 +48,13 @@ const aliases = { cxx: 'cpp', hpp: 'cpp', hxx: 'cpp', + 'c#': 'csharp', + cs: 'csharp', + kt: 'kotlin', + kts: 'kotlin', + pl: 'perl', + rb: 'ruby', + rs: 'rust', '-->': 'plaintext', 'angular-html': 'html', 'angular-ts': 'ts', diff --git a/scripts/measure-size.mjs b/scripts/measure-size.mjs index 016b84a..2deefd1 100644 --- a/scripts/measure-size.mjs +++ b/scripts/measure-size.mjs @@ -98,13 +98,85 @@ const profiles = { languages: ['cmake'], limits: { minified: 6_000, gzip: 2_800, brotli: 2_600 }, }, + csharp: { + source: ` + import { createHighlighter } from './src/core.ts' + import { csharp } from './src/languages/csharp.ts' + globalThis.highlighter = createHighlighter({ languages: [csharp] }) + `, + languages: ['csharp'], + limits: { minified: 7_950, gzip: 3_750, brotli: 3_450 }, + }, + dart: { + source: ` + import { createHighlighter } from './src/core.ts' + import { dart } from './src/languages/dart.ts' + globalThis.highlighter = createHighlighter({ languages: [dart] }) + `, + languages: ['dart'], + limits: { minified: 7_000, gzip: 3_250, brotli: 3_000 }, + }, + java: { + source: ` + import { createHighlighter } from './src/core.ts' + import { java } from './src/languages/java.ts' + globalThis.highlighter = createHighlighter({ languages: [java] }) + `, + languages: ['java'], + limits: { minified: 6_550, gzip: 3_100, brotli: 2_850 }, + }, + kotlin: { + source: ` + import { createHighlighter } from './src/core.ts' + import { kotlin } from './src/languages/kotlin.ts' + globalThis.highlighter = createHighlighter({ languages: [kotlin] }) + `, + languages: ['kotlin'], + limits: { minified: 7_450, gzip: 3_500, brotli: 3_250 }, + }, + lua: { + source: ` + import { createHighlighter } from './src/core.ts' + import { lua } from './src/languages/lua.ts' + globalThis.highlighter = createHighlighter({ languages: [lua] }) + `, + languages: ['lua'], + limits: { minified: 5_650, gzip: 2_750, brotli: 2_550 }, + }, + perl: { + source: ` + import { createHighlighter } from './src/core.ts' + import { perl } from './src/languages/perl.ts' + globalThis.highlighter = createHighlighter({ languages: [perl] }) + `, + languages: ['perl'], + limits: { minified: 7_850, gzip: 3_750, brotli: 3_500 }, + }, + ruby: { + source: ` + import { createHighlighter } from './src/core.ts' + import { ruby } from './src/languages/ruby.ts' + globalThis.highlighter = createHighlighter({ languages: [ruby] }) + `, + languages: ['ruby'], + limits: { minified: 7_700, gzip: 3_650, brotli: 3_350 }, + }, + rust: { + source: ` + import { createHighlighter } from './src/core.ts' + import { rust } from './src/languages/rust.ts' + globalThis.highlighter = createHighlighter({ languages: [rust] }) + `, + languages: ['rust'], + limits: { minified: 6_800, gzip: 3_200, brotli: 2_950 }, + }, all: { source: ` import { defaultHighlighter } from './src/index.ts' globalThis.highlighter = defaultHighlighter `, languages: 'all', - limits: { minified: 30_800, gzip: 10_900, brotli: 9_800 }, + limits: { minified: 50_400, gzip: 17_100, brotli: 15_500 }, }, reactAdapter: { source: `export * from './src/react.ts'`, diff --git a/skills/configure-selective-highlighting/references/languages.md b/skills/configure-selective-highlighting/references/languages.md index 67d3c00..81b375b 100644 --- a/skills/configure-selective-highlighting/references/languages.md +++ b/skills/configure-selective-highlighting/references/languages.md @@ -7,7 +7,9 @@ Import only the definitions the application registers. | Apache | `apache` | `@tanstack/highlight/languages/apache` | - | | CMake | `cmake` | `@tanstack/highlight/languages/cmake` | - | | C++ | `cpp` | `@tanstack/highlight/languages/cpp` | `c++`, `cc`, `cxx`, `hpp`, `hxx` | +| C# | `csharp` | `@tanstack/highlight/languages/csharp` | `c#`, `cs` | | CSS | `css` | `@tanstack/highlight/languages/css` | - | +| Dart | `dart` | `@tanstack/highlight/languages/dart` | - | | Diff | `diff` | `@tanstack/highlight/languages/diff` | `patch` | | Dockerfile | `dockerfile` | `@tanstack/highlight/languages/dockerfile` | `docker` | | EJS | `ejs` | `@tanstack/highlight/languages/ejs` | - | @@ -15,15 +17,21 @@ Import only the definitions the application registers. | Go | `go` | `@tanstack/highlight/languages/go` | `golang` | | HTML | `html` | `@tanstack/highlight/languages/html` | `htm`, `xml`, `angular-html` | | HTTP | `http` | `@tanstack/highlight/languages/http` | - | +| Java | `java` | `@tanstack/highlight/languages/java` | - | | JavaScript | `js` | `@tanstack/highlight/languages/js` | `javascript`, `mjs`, `cjs`, `js-vue` | | JSON | `json` | `@tanstack/highlight/languages/json` | `jsonc`, `json5` | | JSX | `jsx` | `@tanstack/highlight/languages/jsx` | - | +| Kotlin | `kotlin` | `@tanstack/highlight/languages/kotlin` | `kt`, `kts` | +| Lua | `lua` | `@tanstack/highlight/languages/lua` | - | | Markdown | `markdown` | `@tanstack/highlight/languages/markdown` | `md` | | Mermaid | `mermaid` | `@tanstack/highlight/languages/mermaid` | - | | Nginx | `nginx` | `@tanstack/highlight/languages/nginx` | - | +| Perl | `perl` | `@tanstack/highlight/languages/perl` | `pl` | | PHP | `php` | `@tanstack/highlight/languages/php` | - | | Plaintext | `plaintext` | `@tanstack/highlight/languages/plaintext` | `text`, `txt`, `-->` | | Python | `python` | `@tanstack/highlight/languages/python` | `py` | +| Ruby | `ruby` | `@tanstack/highlight/languages/ruby` | `rb` | +| Rust | `rust` | `@tanstack/highlight/languages/rust` | `rs` | | Scheme | `scheme` | `@tanstack/highlight/languages/scheme` | `scm`, `racket` | | Shell | `shell` | `@tanstack/highlight/languages/shell` | `bash`, `sh`, `zsh`, `cmd`, `console` | | SQL | `sql` | `@tanstack/highlight/languages/sql` | - | diff --git a/src/index.ts b/src/index.ts index 2500ca8..17e2a87 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,9 @@ import { import { apache } from './languages/apache.js' import { cmake } from './languages/cmake.js' import { cpp } from './languages/cpp.js' +import { csharp } from './languages/csharp.js' import { css } from './languages/css.js' +import { dart } from './languages/dart.js' import { diff } from './languages/diff.js' import { dockerfile } from './languages/dockerfile.js' import { ejs } from './languages/ejs.js' @@ -16,15 +18,21 @@ import { env } from './languages/env.js' import { go } from './languages/go.js' import { html } from './languages/html.js' import { http } from './languages/http.js' +import { java } from './languages/java.js' import { js } from './languages/js.js' import { json } from './languages/json.js' import { jsx } from './languages/jsx.js' +import { kotlin } from './languages/kotlin.js' +import { lua } from './languages/lua.js' import { markdown } from './languages/markdown.js' import { mermaid } from './languages/mermaid.js' import { nginx } from './languages/nginx.js' +import { perl } from './languages/perl.js' import { php } from './languages/php.js' import { plaintext } from './languages/plaintext.js' import { python } from './languages/python.js' +import { ruby } from './languages/ruby.js' +import { rust } from './languages/rust.js' import { scheme } from './languages/scheme.js' import { shell } from './languages/shell.js' import { sql } from './languages/sql.js' @@ -40,7 +48,9 @@ export type HighlightLanguage = | 'apache' | 'cmake' | 'cpp' + | 'csharp' | 'css' + | 'dart' | 'diff' | 'dockerfile' | 'ejs' @@ -48,15 +58,21 @@ export type HighlightLanguage = | 'go' | 'html' | 'http' + | 'java' | 'js' | 'json' | 'jsx' + | 'kotlin' + | 'lua' | 'markdown' | 'mermaid' | 'nginx' + | 'perl' | 'php' | 'plaintext' | 'python' + | 'ruby' + | 'rust' | 'scheme' | 'shell' | 'sql' @@ -112,7 +128,9 @@ export const allLanguages = [ apache, cmake, cpp, + csharp, css, + dart, diff, dockerfile, ejs, @@ -120,15 +138,21 @@ export const allLanguages = [ go, html, http, + java, js, json, jsx, + kotlin, + lua, markdown, mermaid, nginx, + perl, php, plaintext, python, + ruby, + rust, scheme, shell, sql, diff --git a/src/languages/csharp.ts b/src/languages/csharp.ts new file mode 100644 index 0000000..73f56d8 --- /dev/null +++ b/src/languages/csharp.ts @@ -0,0 +1,142 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { + className: 'attr', + regex: /(?:^[ \t]*|[(,] *)\[[ \t]*(?:[a-z]+: *)?([A-Z]\w*)(?=[ \t]*[\](,])/gm, + group: 1, + }, + { + className: 'keyword', + regex: /(?))/g }, + // Contextual words are keywords only when another word follows, as in `record Person` or `select u`. + { + className: 'keyword', + regex: /\b(?:async|by|equals|file|from|group|into|join|let|on|partial|record|required|scoped|select)\b(?=\s+(?!(?:in|is|as)\b)[\w@(])/g, + }, + { className: 'literal', regex: /\b(?:true|false|null)\b/g }, + { + className: 'type', + regex: /\b(?:bool|byte|char|decimal|double|dynamic|float|int|long|nint|nuint|object|sbyte|short|string|uint|ulong|ushort|void)\b/g, + }, + { className: 'type', regex: /\b(?:class|struct|interface|enum|record(?:\s+struct)?|new|using(?:\s+static)?|namespace)\s+([A-Za-z_][\w.]*)/g, group: 1 }, + { className: 'type', regex: /\busing\s+\w+\s*=\s*([A-Za-z_][\w.]*)/g, group: 1 }, + { className: 'function', regex: /\b[A-Za-z_]\w*(?=\s*\(|<(?:[\w\s,.?[\]]|<[\w\s,.?[\]]{0,40}>){0,40}>\s*\()/g }, + { className: 'property', regex: /(? match[1] ? 'property' : 'type', + regex: /\b[A-Z][A-Z\d_]*[a-z]\w*(?=(\s*(?:=[^=>]|\{\s*(?:get|set|init)\b)|(?<=[{,(]\s*\w+)\s*:[^:]|(?<=[\w>?\]] +\w+)\s*=>)?)/g, + }, + { + className: 'number', + regex: /(?:^|\.\.|[^\w.])((?:0[xX][\da-fA-F_]+|0[bB][01_]+|(?:\d[\d_]*(?:\.\d[\d_]*)?|\.\d[\d_]*)(?:[eE][+-]?\d[\d_]*)?)(?:[uUlL]{1,2}|[fFdDmM])?)(?!\w|\.\d)/g, + group: 1, + }, + { className: 'operator', regex: /=>|\?\?=?|\?\.|::|\.\.|->|<<=?|>>>?=?|\+\+|--|&&|\|\||[+\-*/%&|^!<>=]=?|[~?:]/g }, +] satisfies Parameters[1] + +export const csharp = defineLanguage({ + name: 'csharp', + aliases: ['c#', 'cs'], + tokenize: (code) => collectPatternRanges(code, patterns, scanCsharp(code)), +}) + +function scanCsharp(code: string) { + const ranges: Array = [] + const lexical = /\/\/|\/\*|['"]|^[ \t]*#/gm + let end = 0 + let match: RegExpExecArray | null + while ((match = lexical.exec(code))) { + const token = match[0] + let start = match.index + let className: 'comment' | 'meta' | 'string' = 'comment' + if (token === '/*') { + const close = code.indexOf('*/', start + 2) + end = close < 0 ? code.length : close + 2 + } else if (token === '//') { + end = lineEnd(code, start) + } else if (token === "'") { + const close = charEnd(code, start) + if (!close) continue + end = close + className = 'string' + } else if (token === '"') { + start = prefixStart(code, start, end) + end = stringEnd(code, start, match.index, 0) + className = 'string' + } else { + start += token.length - 1 + end = lineEnd(code, start) + className = 'meta' + } + ranges.push({ start, end, className }) + lexical.lastIndex = end + } + return ranges +} + +function lineEnd(code: string, start: number) { + const index = code.indexOf('\n', start) + return index < 0 ? code.length : index +} + +function prefixStart(code: string, index: number, min: number) { + while (index > min && (code[index - 1] === '$' || code[index - 1] === '@')) index-- + return index +} + +function charEnd(code: string, start: number) { + for (let index = start + 1; index < code.length && index < start + 12; index++) { + if (code[index] === '\\') index++ + else if (code[index] === '\n') return 0 + else if (code[index] === "'") return index + 1 + } + return 0 +} + +function stringEnd(code: string, start: number, quote: number, level: number): number { + if (level > 16) return code.length + const prefix = code.slice(start, quote) + const verbatim = prefix.includes('@') + const interpolated = prefix.includes('$') + let index = quote + while (code[index] === '"') index++ + const count = index - quote + if (count === 2) return index + if (count > 2 && !verbatim) { + // Raw strings close on the same number of quotes; holes rarely contain that many. + const close = code.indexOf('"'.repeat(count), index) + return close < 0 ? code.length : close + count + } + index = quote + 1 + let depth = 0 + while (index < code.length) { + const char = code[index] + if (char === '\n' && !verbatim) return index + if (depth) { + if (char === "'") { + index = charEnd(code, index) || index + 1 + continue + } + if (char === '"') { + index = stringEnd(code, prefixStart(code, index, 0), index, level + 1) + continue + } + if (char === '{') depth++ + else if (char === '}') depth-- + } else if (char === '\\' && !verbatim) { + index++ + } else if (char === '"') { + if (!verbatim || code[index + 1] !== '"') return index + 1 + index++ + } else if (char === '{' && interpolated) { + if (code[index + 1] === '{') index++ + else depth = 1 + } + index++ + } + return code.length +} diff --git a/src/languages/dart.ts b/src/languages/dart.ts new file mode 100644 index 0000000..4a46589 --- /dev/null +++ b/src/languages/dart.ts @@ -0,0 +1,80 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { className: 'attr', regex: /@[A-Za-z_$][\w$]*/g }, + // Named-argument and record labels; the lookbehind keeps ternaries plain. + { className: 'property', regex: /(?|~\/=?|<<=?|>>>?=?|&&|\|\||\+\+|--|[+\-*/%&|^!<>=]=?|[~?:]/g }, +] satisfies Parameters[1] + +export const dart = defineLanguage({ + name: 'dart', + tokenize(code) { + const ranges: Array = [] + const lexical = /\/\/.*|\/\*|(?:(? = [] + let open = true + while (index < code.length) { + const char = code[index]! + if (open) { + const quote = code.startsWith(char.repeat(3), index) ? char.repeat(3) : char + stack.push(quote) + index += quote.length + open = false + continue + } + const top = stack[stack.length - 1] + if (top === undefined) break + if (typeof top === 'number') { + if (char === '"' || char === "'") { + open = true + raw = code[index - 1] === 'r' + continue + } + if (char === '}' && !top) stack.pop() + else if (char === '{' || char === '}') stack[stack.length - 1] = top + (char === '{' ? 1 : -1) + index++ + } else if (code.startsWith(top, index) || (char === '\n' && top.length < 3)) { + if (char !== '\n') index += top.length + stack.pop() + raw = false + } else if (raw) index++ + else if (char === '\\') index += 2 + else if (char === '$' && code[index + 1] === '{') { + stack.push(0) + index += 2 + } else index++ + } + return Math.min(index, code.length) +} diff --git a/src/languages/index.ts b/src/languages/index.ts index 6bf5cb1..83f875d 100644 --- a/src/languages/index.ts +++ b/src/languages/index.ts @@ -1,7 +1,9 @@ export { apache } from './apache.js' export { cmake } from './cmake.js' export { cpp } from './cpp.js' +export { csharp } from './csharp.js' export { css } from './css.js' +export { dart } from './dart.js' export { diff } from './diff.js' export { dockerfile } from './dockerfile.js' export { ejs } from './ejs.js' @@ -9,15 +11,21 @@ export { env } from './env.js' export { go } from './go.js' export { html } from './html.js' export { http } from './http.js' +export { java } from './java.js' export { js } from './js.js' export { json } from './json.js' export { jsx } from './jsx.js' +export { kotlin } from './kotlin.js' +export { lua } from './lua.js' export { markdown } from './markdown.js' export { mermaid } from './mermaid.js' export { nginx } from './nginx.js' +export { perl } from './perl.js' export { php } from './php.js' export { plaintext } from './plaintext.js' export { python } from './python.js' +export { ruby } from './ruby.js' +export { rust } from './rust.js' export { scheme } from './scheme.js' export { shell } from './shell.js' export { sql } from './sql.js' diff --git a/src/languages/java.ts b/src/languages/java.ts new file mode 100644 index 0000000..fc85b79 --- /dev/null +++ b/src/languages/java.ts @@ -0,0 +1,48 @@ +import { defineLanguage } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' +import type { Pattern } from '../internal/patterns.js' + +const lexical: Array = [ + { + className: (match) => match[0].startsWith('/') ? 'comment' : 'string', + regex: /\/\/[^\n]*|\/\*[\s\S]*?(?:\*\/|$)|"""(?:\\[\s\S]|[^\\])*?(?:"""|\\?$)|"(?:\\.|[^"\\\n])*\\?"?|'(?:\\.[^'\n]{0,5}|[^'\\\n])'/g, + }, +] + +const patterns: Array = [ + { + className: (match) => match[0] === '@interface' ? 'keyword' : 'attr', + regex: /@[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*/g, + }, + { + className: 'keyword', + regex: /\b(?:abstract|assert|break|case|catch|class|const|continue|default|do|else|enum|extends|final|finally|for|goto|if|implements|import|instanceof|interface|native|new|non-sealed|package|permits|private|protected|public|return|sealed|static|strictfp|super|switch|synchronized|this|throws?|transient|try|volatile|while)\b|\b(?:record|var|when|yield)\b(?=[ \t]+[\w"'(!-])|^[ \t]*(?:(?:open[ \t]+)?module|requires(?:[ \t]+transitive)?|exports|opens|uses|provides)\b(?=[ \t]+[A-Za-z_])/gm, + }, + { className: 'literal', regex: /\b(?:true|false|null)\b/g }, + { className: 'type', regex: /\b(?:boolean|byte|char|double|float|int|long|short|void)\b/g }, + { className: 'type', regex: /(?>>=?|->|::|<<=?|>>=?|\+\+|--|&&|\|\||[+\-*/%&|^!<>=]=?|[~?:]/g }, +] + +export const java = defineLanguage({ + name: 'java', + tokenize(code) { + const occupied = new Uint8Array(code.length) + const ranges = collectPatternRanges(code, lexical, [], occupied) + // Leave package/import paths plain instead of colouring each segment as a property. + for (const match of code.matchAll(/^([ \t]*(?:package|import)[ \t]+(?:static[ \t]+)?)[\w.*]+/gm)) { + const start = match.index + match[1].length + if (!occupied[match.index]) occupied.fill(1, start, match.index + match[0].length) + } + return collectPatternRanges(code, patterns, ranges, occupied) + }, +}) diff --git a/src/languages/kotlin.ts b/src/languages/kotlin.ts new file mode 100644 index 0000000..84a7a80 --- /dev/null +++ b/src/languages/kotlin.ts @@ -0,0 +1,88 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { className: 'attr', regex: /(?[\w.?*, ]*)?>)?[ \t]*[({])|(?<=^[ \t]*|[{=][ \t]*)[A-Z][A-Z\d_]*[a-z]\w*(?=[ \t]*\{)/gm }, + { className: 'type', regex: /\b[A-Z][A-Z\d_]*[a-z]\w*/g }, + { className: 'number', regex: /(?|::|&&|\|\||\+\+|--|[+\-*/%!<>=]=?/g }, +] satisfies Parameters[1] + +export const kotlin = defineLanguage({ + name: 'kotlin', + aliases: ['kt', 'kts'], + tokenize(code) { + const ranges: Array = [] + const occupied = new Uint8Array(code.length) + const lexical = /\/\/.*|\/\*|"""|"|'(?:\\(?:u[\da-fA-F]{4}|.)|[^'\\\n])'|`[^`\n]+`(\s*\()?/g + let match: RegExpExecArray | null + while ((match = lexical.exec(code))) { + const token = match[0] + let from = match.index + let end = lexical.lastIndex + let className: TokenRange['className'] = token[0] === '/' ? 'comment' : 'string' + if (token === '/*') { + const nested = /\/\*|\*\//g + nested.lastIndex = end + for (let depth = 1; depth && (match = nested.exec(code));) depth += match[0] === '/*' ? 1 : -1 + end = match ? nested.lastIndex : code.length + } else if (token[0] === '`') { + if (!match[1]) { + occupied.fill(1, from, end) + continue + } + end = from + token.lastIndexOf('`') + 1 + className = 'function' + } else if (token[0] === '"') { + while (code[from - 1] === '$') from-- + end = stringEnd(code, end, token.length > 1) + } + ranges.push({ start: from, end, className }) + lexical.lastIndex = end + } + for (const path of code.matchAll(/^[ \t]*(?:package|import)[ \t]+([\w.*`]+)/gm)) { + occupied.fill(1, path.index + path[0].length - path[1].length, path.index + path[0].length) + } + return collectPatternRanges(code, patterns, ranges, occupied) + }, +}) + +// Stack of open strings (raw flag) and `${` templates (brace depth), so nested quotes stay inside. +function stringEnd(code: string, index: number, raw: boolean) { + const stack: Array = [raw] + while (index < code.length) { + const top = stack[stack.length - 1] + const char = code[index++] + if (typeof top === 'number') { + if (char === '"') { + const nested = code.startsWith('""', index) + stack.push(nested) + if (nested) index += 2 + } else if (char === "'") index = code.indexOf("'", index + (code[index] === '\\' ? 2 : 1)) + 1 || code.length + else if (char === '{') stack[stack.length - 1] = top + 1 + else if (char === '}') { + if (top) stack[stack.length - 1] = top - 1 + else stack.pop() + } + } else if (char === '"' && (!top || code.startsWith('""', index)) || char === '\n' && !top) { + if (char === '\n') index-- + else if (top) while (code[index] === '"') index++ + stack.pop() + if (!stack.length) return index + } else if (char === '\\' && !top) index++ + else if (char === '$' && code[index] === '{') { + stack.push(0) + index++ + } + } + return code.length +} diff --git a/src/languages/lua.ts b/src/languages/lua.ts new file mode 100644 index 0000000..d5bf266 --- /dev/null +++ b/src/languages/lua.ts @@ -0,0 +1,24 @@ +import { defineLanguage } from '../core.js' +import { patternTokenizer } from '../internal/patterns.js' + +export const lua = defineLanguage({ + name: 'lua', + tokenize: patternTokenizer([ + { + className: (match) => match[0][0] === '-' ? 'comment' : match[0][0] === '#' ? 'meta' : 'string', + regex: /^#![^\n]*|--(?:\[(=*)\[[\s\S]*?(?:\]\1\]|$)|[^\n]*)|\[(=*)\[[\s\S]*?(?:\]\2\]|$)|(["'`])(?:\\z\s*|\\\r?\n|\\[\s\S]|(?!\3)[^\\\n])*\3?/g, + }, + { + className: 'keyword', + regex: /\b(?:and|break|do|else|elseif|end|for|function|goto|if|in|local|not|or|repeat|return|then|until|while)\b|\b(?<=<\s*)(?:const|close)(?=\s*>)/g, + }, + { className: 'literal', regex: /\b(?:nil|true|false)\b/g }, + { className: 'function', regex: /\b[A-Za-z_]\w*(?=\s*\(|[ \t]*["'{]|[ \t]*\[=*\[|\s*=\s*function\b)/g }, + { + className: 'number', + regex: /(?:(?]=|\/\/|<<|>>|::|[-+*/%^#&|~<>=]/g }, + ]), +}) diff --git a/src/languages/perl.ts b/src/languages/perl.ts new file mode 100644 index 0000000..4963c93 --- /dev/null +++ b/src/languages/perl.ts @@ -0,0 +1,151 @@ +import { defineLanguage, type HighlightTokenClass, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { className: 'property', regex: /\b[A-Za-z_]\w*(?=\s*=>)|(?<=[$+\w\]}]\{|->\{)-?[A-Za-z_]\w*(?=\})/g }, + { className: 'function', regex: /(?<=->|\bsub\s+)[A-Za-z_][\w:]*/g }, + { className: 'type', regex: /(?<=\b(?:package|use|no|require)\s+)(?!v\d)[A-Za-z_]\w*(?:::\w+)*|\b[A-Z]\w*(?:::\w+)*(?=\s*->)/g }, + { className: 'keyword', regex: /\b(?:my|our|local|state|sub|package|use|no|require|if|elsif|else|unless|while|until|for(?:each)?|do|last|next|redo|return|and|or|not|xor|cmp|eq|ne|lt|gt|le|ge|x|eval|try|catch|finally|BEGIN|END)\b/g }, + { className: 'literal', regex: /\b(?:undef|__PACKAGE__|__FILE__|__LINE__|__SUB__)\b/g }, + { className: 'function', regex: /\b(?:print|say|die|bless)\b|\b[A-Za-z_]\w*(?=\s*\()/g }, + { className: 'number', regex: /(?|->|=>|::|\.\.\.|[=!]~|([-+*/%.&|^<>])\1?=?|[!=]=?|[~?:\\]/g }, +] satisfies Parameters[1] + +export const perl = defineLanguage({ + name: 'perl', + aliases: ['pl'], + tokenize: (code) => collectPatternRanges(code, patterns, scanPerl(code)), +}) + +const variable = /\$\$(?![\w{$:])|[$@%&]\$*#?(?:\{\^?\w+\}|\^\w|(?:::)?\w+(?:::\w+)*|[*+-]|(?=[{$]))|\$[!@&`'"+/\\,;.<>|?~^-]?/y +const word = /[A-Za-z_]\w*(?:::\w+)*/y +const space = /\s*/y +const flags = /[a-z]*/y +const heredoc = /<<(~?)(["']?)([A-Za-z_]\w*)\2/y +const brackets = /[{(<[]/ + +function scanPerl(code: string) { + const ranges: Array = [] + const pending: Array = [] + // `value`: the previous token ends an operand, so `/` divides and `%`/`&` are operators. + // `last`: text of the previous word or operator (`->`, `sub`, a list operator before `%h`). + let value = false + let last = '' + // Keeps unclosed single-line scans linear: once one fails, every later start with the same + // delimiter fails before that newline (`>>> 0` turns "no newline" into end of input). + const failed: Record = {} + const line = (start: number) => { + if (start < failed[code[start]]) return -1 + const end = quoted(code, start, true) + if (end < 0) failed[code[start]] = code.indexOf('\n', start) >>> 0 + return end + } + let i = 0 + while (i < code.length) { + const char = code[i] + const start = i + const lineStart = (code[i - 1] || '\n') === '\n' + let className: HighlightTokenClass | undefined = 'string' + let match: RegExpExecArray | null + variable.lastIndex = word.lastIndex = heredoc.lastIndex = i + if (char === '\n' && pending.length) { + for (const terminator of pending.splice(0)) { + terminator.lastIndex = i + 1 + ranges.push({ start: i + 1, end: (i = terminator.exec(code) ? terminator.lastIndex : code.length), className }) + } + continue + } + if (/\s/.test(char)) { + i++ + continue + } + if (char === '=' && lineStart && /[A-Za-z]/.test(code[i + 1])) { + i = skip(/[^]*?^=cut\b.*|[^]*/my, code, i) + className = 'comment' + } else if (char === '#') { + i = code.indexOf('\n', i) + if (i < 0) i = code.length + className = start || code[1] !== '!' ? 'comment' : 'meta' + } else if (/["'`]/.test(char)) { + i = quoted(code, i) + } else if ( + /[$@%&]/.test(char) && variable.test(code) && + // After an operand `%`/`&` are operators, unless it is a list-operator word or block (`keys %h`, `map {...} %h`). + (!value || /[\w}]/.test(last) || /[$@]/.test(char)) + ) { + i = variable.lastIndex + className = 'variable' + } else if ((match = word.exec(code))) { + const name = match[0] + const after = skip(space, code, (i = word.lastIndex)) + const open = code[after] || '' + className = undefined + if (lineStart && /^__(?:END|DATA)__$/.test(name)) { + i = code.length + className = 'comment' + } else if ( + // Not a method/sub name; unspaced, the next char must not end a bareword key or label (`y => 1`, `$h{s}`); + // spaced, only a bracket or `/` opens. + /^(?:q[qwrx]?|m|s|tr|y)$/.test(name) && last !== '->' && last !== 'sub' && + (after > i ? /[{(<[/]/.test(open) : !/[\w\s=,;)}\]>:.-]/.test(open)) + ) { + let end = !brackets.test(open) && name[0] !== 'q' ? line(after) : quoted(code, after) + if (end > 0 && end < code.length && /^(?:s|tr|y)$/.test(name)) { + // Non-bracket: the middle delimiter closes the pattern and opens the replacement. + const next = skip(space, code, end) + end = !brackets.test(open) ? line(end - 1) : brackets.test(code[next]) ? quoted(code, next) : -1 + } + if (end > 0) { + i = skip(flags, code, end) + className = 'string' + } + } + last = name + value = !/^(?:split|if|unless|elsif|while|until|and|or|not|return|grep|map|x|eq|ne|lt|gt|le|ge|cmp)$/.test(name) + } else if (/\d/.test(char)) { + i = skip(/\d\w*(?:\.\d\w*)*/y, code, i) + className = undefined + last = '' + value = true + } else if ((match = heredoc.exec(code))) { + i = heredoc.lastIndex + pending.push(new RegExp(`^${match[1] && '[ \\t]*'}${match[3]}$`, 'gm')) + } else if (char !== '/' || value || (i = line(i)) < 0) { + // Consume `->`, `&&`, `//` whole so their second char is not read as a sigil or regex opener. + i = start + (/->|&&|\/\//.test(code.slice(start, start + 2)) ? 2 : 1) + className = undefined + value = /[)\]}]/.test(char) + last = code.slice(start, i) + continue + } else i = skip(flags, code, i) + if (className) { + ranges.push({ start, end: i, className }) + if (className === 'comment') continue + last = '' + value = true + } + } + return ranges +} + +function skip(regex: RegExp, code: string, index: number) { + regex.lastIndex = index + regex.exec(code) + return regex.lastIndex +} + +// Returns the index after the closing delimiter, or -1 when `single` and the line ends first. +function quoted(code: string, start: number, single = false) { + const open = code[start] + const close = ')]}>'['([{<'.indexOf(open)] || open + let depth = 0 + for (let index = start + 1; index < code.length; index++) { + const char = code[index] + if (single && char === '\n') return -1 + if (char === '\\') index++ + else if (char === close && !depth--) return index + 1 + else if (char === open && open !== close) depth++ + } + return single ? -1 : code.length +} diff --git a/src/languages/ruby.ts b/src/languages/ruby.ts new file mode 100644 index 0000000..11dc711 --- /dev/null +++ b/src/languages/ruby.ts @@ -0,0 +1,109 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { className: 'variable', regex: /@@?[A-Za-z_]\w*|\$(?:\w+|[^\s\w])/g }, + { className: 'string', regex: /(?!~^&|[\]]+)/g, group: 1 }, + { className: 'keyword', regex: /(?|===?|=~|!~|\*\*=?|&\.|\|\|=?|&&=?|->|=>|\.\.\.?|::|<<=?|>>=?|!=|[+\-*/%&^<>=]=?|(?[1] + +export const ruby = defineLanguage({ + name: 'ruby', + aliases: ['rb'], + tokenize: (code) => collectPatternRanges(code, patterns, scanRuby(code)), +}) + +const pairs: Record = { '(': ')', '[': ']', '{': '}', '<': '>' } + +function scanRuby(code: string) { + const ranges: Array = [] + const lexical = /\$.|^=begin\b|[#'"`/]|%[qQwWiIrsx]?[^\w\s]|<<[~-]?(['"`]?)([A-Za-z_]\w*)\1/gm + let lineEnd = 0 + let bodyEnd = 0 + let match: RegExpExecArray | null + while ((match = lexical.exec(code))) { + const token = match[0] + const char = token[0] + let from = match.index + let end = lexical.lastIndex + let className: 'comment' | 'meta' | 'string' = 'string' + if (from > lineEnd && from < bodyEnd) { + lexical.lastIndex = bodyEnd + continue + } + if (char === '$') continue + lexical.lastIndex = from + 1 + if (char === '#') { + end = code.indexOf('\n', from) + if (end < 0) end = code.length + className = from || code[1] !== '!' ? 'comment' : 'meta' + } else if (char === '=') { + const close = /^=end\b.*/gm + close.lastIndex = end + end = close.exec(code) ? close.lastIndex : code.length + className = 'comment' + } else if (char === '<') { + // Bare `< from ? bodyEnd : newline) + 1 + const close = new RegExp(`^[\\t ]*${match[2]}\\r?$`, 'gm') + close.lastIndex = start + bodyEnd = close.exec(code) ? close.lastIndex : code.length + ranges.push({ start, end: bodyEnd, className }) + lineEnd = newline + } else if (char === '/' || char === '%') { + if (isValue(code, from)) continue + const delimiter = token[token.length - 1] + end = stringEnd(code, end, pairs[delimiter] || delimiter, pairs[delimiter] ? delimiter : '', !/[qwis]/.test(token[1] || ''), char === '/' ? 1 : 0) + // `total /count * 100 / 2`: a regex that hugs its opener but ends in a space is division. + if (end < 0 || char === '/' && end < code.length && code[from + 1] !== ' ' && code[end - 2] === ' ') continue + if (char === '/' || token[1] === 'r') while (/[a-z]/.test(code[end] || '')) end++ + } else { + end = stringEnd(code, end, char, '', char !== "'", 0) + if (code[from - 1] === ':' && !/[\w:]/.test(code[from - 2] || '')) from-- + } + ranges.push({ start: from, end, className }) + lexical.lastIndex = end + } + return ranges +} + +function isValue(code: string, index: number) { + let start = index + while (code[start - 1] === ' ' || code[start - 1] === '\t') start-- + if (!/[\w)\]}"'`]/.test(code[start - 1] || '')) return false + let word = start + while (/\w/.test(code[word - 1] || '')) word-- + // `split /,/` and `puts %w[a]` are arguments; `a / b`, `x /= 2` and `w /2` are operators. + return !(start < index && /^[A-Za-z_]/.test(code.slice(word, start)) && /[^\s=\d]/.test(code[index + 1] || ' ')) +} + +/** + * Returns the index just past `close`, or -1 when a single-line regex hits a newline. + * `open` (paired delimiters only) nests; `interpolate` enables `#{}`; `mode` 1 is a `/regex/`, + * mode 2 is interpolated code where quotes open nested strings; `nest` caps interpolation depth. + */ +function stringEnd(code: string, index: number, close: string, open: string, interpolate: boolean, mode: number, nest = 0): number { + let depth = 0 + while (index < code.length) { + const char = code[index++] + if (char === '\\') index++ + else if (mode === 1 && char === '\n') return -1 + else if (mode === 2 && /["'`]/.test(char)) index = stringEnd(code, index, char, '', char !== "'", 0, nest + 1) + else if (mode === 2 && char === '/' && !isValue(code, index - 1)) index = Math.max(index, stringEnd(code, index, '/', '', true, 1, nest + 1)) + else if (interpolate && nest < 9 && char === '#' && code[index] === '{') index = stringEnd(code, index + 1, '}', '{', false, 2, nest) + else if (char === open) depth++ + else if (char === close && !depth--) return index + } + return code.length +} diff --git a/src/languages/rust.ts b/src/languages/rust.ts new file mode 100644 index 0000000..b99b8d3 --- /dev/null +++ b/src/languages/rust.ts @@ -0,0 +1,83 @@ +import { defineLanguage, type HighlightTokenClass, type TokenRange } from '../core.js' +import { collectPatternRanges } from '../internal/patterns.js' + +const patterns = [ + { className: 'meta', regex: /^#!(?!\s*\[).*/g }, + { className: 'attr', regex: /#!?\[\s*[A-Za-z_][\w:]*/g }, + { className: 'function', regex: /\bmacro_rules!|\b[A-Za-z_]\w*!(?=\s*[([{])/g }, + { + className: 'keyword', + regex: /\b(?|->|::|\.\.[.=]?|<<=?|>>=?|&&|\|\||[+\-*/%&|^!<>=]=?|\?/g }, +] satisfies Parameters[1] + +const charLiteral = /'(?:\\(?:u\{[\da-fA-F_]*\}|x[\da-fA-F]{2}|[^\n])|[^\\'\n\r])'/uy +const lifetime = /'[A-Za-z_]\w*/y + +export const rust = defineLanguage({ + name: 'rust', + aliases: ['rs'], + tokenize: (code) => collectPatternRanges(code, patterns, scanRust(code)), +}) + +function scanRust(code: string) { + const ranges: Array = [] + const lexical = /\/[/*]|(?= 0) end = close + 1 + match[1].length + } else { + for (let index = start + token.length; index < code.length; index++) { + if (code[index] === '\\') index++ + else if (code[index] === '"') { + end = index + 1 + break + } + } + } + } + ranges.push({ start, end, className }) + lexical.lastIndex = end + } + return ranges +} diff --git a/test/csharp.test.ts b/test/csharp.test.ts new file mode 100644 index 0000000..a02f11c --- /dev/null +++ b/test/csharp.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { csharp } from '../src/languages/csharp' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [csharp] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'csharp') + +const showcase = `#nullable enable +using System.Text.Json; +namespace Demo.Api; + +/// Stores users. Don't "panic". +[ApiController] +[Route("api/[controller]")] +public sealed partial class UsersController(IUserStore store) : ControllerBase +{ + #region Queries + [HttpGet("{id:int}")] + public async Task> Get(int id, CancellationToken ct = default) + { + var user = await store.FindAsync(id, ct)!; + if (user is not { Active: true }) return NotFound($"No user {id}"); + var path = @"C:\\data\\users\\" + user.Name; + var json = $$"""{"id": {{user.Id}}, "tags": [1, 2]}"""; + var label = $"{(user.Admin ? "Admin" : "User")}: {user.Balance:N2} {{literal}}"; + long big = 1_000_000L; + user.Nickname ??= user?.Name ?? "anon"; + Console.WriteLine("http://example.com // not a comment"); + return Ok(user); + } + #endregion + + public required string Name { get; init; } + public record Person(string First, int Age); + + public IEnumerable Adults(IEnumerable people) => + from p in people + where p.Age >= 18 + orderby p.First descending + select p.First; + + T Parse(string s) where T : notnull => JsonSerializer.Deserialize(s)!; + (int a, string b) Pair() => (1, "x"); + Dictionary> map = new(); + int[] values = [1, 2, 3]; +} +` + +describe('C# documentation syntax', () => { + it('highlights a realistic controller sample', () => { + const code = showcase + expect(classes(code, '#nullable enable')).toEqual(['meta']) + expect(classes(code, 'System.Text.Json')).toEqual(['type']) + expect(classes(code, '/// ')).toEqual(['comment']) + expect(classes(code, 'ApiController')).toEqual(['attr']) + expect(classes(code, 'Route')).toEqual(['attr']) + expect(classes(code, '"api/[controller]"')).toEqual(['string']) + expect(classes(code, 'partial class UsersController')).toEqual(['keyword', undefined, 'keyword', undefined, 'type']) + expect(classes(code, '#region Queries')).toEqual(['meta']) + expect(classes(code, 'HttpGet')).toEqual(['attr']) + expect(classes(code, 'Task(')).toEqual(['function', 'operator', undefined, 'operator', undefined]) + expect(classes(code, 'where T : notnull =>')).toEqual(['keyword', undefined, 'operator', undefined, 'keyword', undefined, 'operator']) + expect(classes(code, 'Deserialize(s)!')).toEqual(['function', 'operator', undefined, 'operator', undefined, 'operator']) + expect(classes(code, 'Dictionary>')).toEqual(['type', 'operator', 'type', undefined, 'type', 'operator', 'type', 'operator']) + expect(classes(code, '[1, 2, 3]')).toEqual([undefined, 'number', undefined, 'number', undefined, 'number', undefined]) + }) + + it('scans regular, verbatim and character literals', () => { + const code = `var a = "a \\" b // c";\nvar p = @"C:\\path\\" + @"say ""hi""\nnext line";\nchar q = '\\''; char u = '\\u0041'; char d = '"';\nvar e = "";` + expect(classes(code, '"a \\" b // c"')).toEqual(['string']) + expect(classes(code, '@"C:\\path\\"')).toEqual(['string']) + expect(classes(code, '+ @"say')).toEqual(['operator', undefined, 'string']) + expect(classes(code, '@"say ""hi""\nnext line"')).toEqual(['string']) + expect(classes(code, "'\\''")).toEqual(['string']) + expect(classes(code, "'\\u0041'")).toEqual(['string']) + expect(classes(code, `'"'`)).toEqual(['string']) + expect(classes(code, '""')).toEqual(['string']) + }) + + it('finds the end of interpolated strings with nested quotes and braces', () => { + for (const string of [ + '$"Hello {name}"', + '$"{dict["key"]}"', + '$"{(ok ? "yes" : "no")}"', + '$"{value:N2} {{literal}} }}"', + `$"{'}'} {c}"`, + '$"{$"inner {x}"} and {y}"', + '$@"{path}\\{file}"', + '@$"C:\\{dir}\\"', + '$@"say ""{x}"""', + ]) { + const code = `var s = ${string}; var after = "x";` + expect(classes(code, string), string).toEqual(['string']) + expect(classes(code, 'var after'), string).toEqual(['keyword', undefined]) + } + }) + + it('stops unterminated single-line strings at the line end', () => { + const code = 'var a = "open\nvar b = $"{open\nvar c = 1;' + expect(classes(code, 'var b')).toEqual(['keyword', undefined]) + expect(classes(code, 'var c = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + }) + + it('matches raw string delimiters by quote count', () => { + const multi = '"""\n He said "hi" // and ""quoted""\n """' + for (const string of [multi, '""""a """ b""""', '$"""{x} "q" """', '$$"""{{x}} {not a hole}"""']) { + const code = `var s = ${string};\nvar after = 1;` + expect(classes(code, string), string).toEqual(['string']) + expect(classes(code, 'var after'), string).toEqual(['keyword', undefined]) + } + }) + + it('keeps comments and preprocessor lines apart from strings', () => { + const code = `/* block "quote' */ var j = 1;\n// it's "fine"\n/// doc\n#if DEBUG\n Log("#if not meta");\n#endif\n#pragma warning disable CS0168\nvar url = "http://x"; // tail` + expect(classes(code, `/* block "quote' */`)).toEqual(['comment']) + expect(classes(code, 'var j')).toEqual(['keyword', undefined]) + expect(classes(code, `// it's "fine"`)).toEqual(['comment']) + expect(classes(code, '/// doc')).toEqual(['comment']) + expect(classes(code, '#if DEBUG')).toEqual(['meta']) + expect(classes(code, '"#if not meta"')).toEqual(['string']) + expect(classes(code, '#endif')).toEqual(['meta']) + expect(classes(code, '#pragma warning disable CS0168')).toEqual(['meta']) + expect(classes(code, '"http://x"')).toEqual(['string']) + expect(classes(code, '// tail')).toEqual(['comment']) + }) + + it('detects line-leading and parameter attributes', () => { + const code = `[assembly: InternalsVisibleTo("X")]\n[Required, MaxLength(50)]\nitems[0] = Lookup[Key];\nint[] xs =\n[\n Foo,\n];\nvar ys = [Bar, Baz];` + expect(classes(code, 'assembly: InternalsVisibleTo')).toEqual([undefined, 'operator', undefined, 'attr']) + expect(classes(code, 'Required')).toEqual(['attr']) + expect(classes(code, 'MaxLength')).toEqual(['function']) + expect(classes(code, 'items[0]')).toEqual([undefined, 'number', undefined]) + expect(classes(code, 'Lookup[Key]')).toEqual(['type', undefined, 'type', undefined]) + expect(classes(code, 'Foo')).toEqual(['type']) + expect(classes(code, '[Bar')).toEqual([undefined, 'type']) + }) + + it('separates keywords, contextual words, literals and types', () => { + const code = `var record = 5; var file = record + 1; var select = from;\nfile sealed class Hidden { }\npublic int Count { get; set; }\nvoid M() { this.x = base.y; nameof(x); }\nbool ok = true || false; object? o = null;\nnint n; nuint m; dynamic d; MAX_SIZE = 1;\nif (x is not null and > 0 or < -1) { }\nvar n2 = new Foo { Bar = 1 };` + expect(classes(code, 'var record = 5')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes(code, 'var file = record + 1')).toEqual(['keyword', undefined, 'operator', undefined, 'operator', undefined, 'number']) + expect(classes(code, 'var select = from;')).toEqual(['keyword', undefined, 'operator', undefined]) + expect(classes(code, 'file sealed class Hidden')).toEqual(['keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'type']) + expect(classes(code, 'Count { get; set; }')).toEqual(['property', undefined, 'keyword', undefined, 'keyword', undefined]) + expect(classes(code, 'this.x = base.y')).toEqual(['keyword', undefined, 'property', undefined, 'operator', undefined, 'keyword', undefined, 'property']) + expect(classes(code, 'nameof(')).toEqual(['keyword', undefined]) + expect(classes(code, 'true || false')).toEqual(['literal', undefined, 'operator', undefined, 'literal']) + expect(classes(code, 'object? o = null')).toEqual(['type', 'operator', undefined, 'operator', undefined, 'literal']) + expect(classes(code, 'nint n; nuint m; dynamic d;')).toEqual(['type', undefined, 'type', undefined, 'type', undefined]) + expect(classes(code, 'MAX_SIZE')).toEqual([undefined]) + expect(classes(code, 'is not null and > 0 or')).toEqual(['keyword', undefined, 'keyword', undefined, 'literal', undefined, 'keyword', undefined, 'operator', undefined, 'number', undefined, 'keyword']) + const patterns = 'var r = s switch { Circle { Radius: > 0 } => F(Name: 1), _ => ok ? Yes : No };\nvoid M() where TKey : notnull { }' + expect(classes(patterns, 'Radius: >')).toEqual(['property', 'operator', undefined, 'operator']) + expect(classes(patterns, 'Name:')).toEqual(['property', 'operator']) + expect(classes(patterns, 'Yes : No')).toEqual(['type', undefined, 'operator', undefined, 'type']) + expect(classes(patterns, 'TKey : notnull')).toEqual(['type', undefined, 'operator', undefined, 'keyword']) + expect(classes(code, 'new Foo { Bar = 1 }')).toEqual(['keyword', undefined, 'type', undefined, 'property', undefined, 'operator', undefined, 'number', undefined]) + }) + + it('classifies numbers and operators', () => { + for (const number of ['1_000', '0xFF', '0x_FF_FF', '0b1010_1010', '1.5f', '1.5m', '100L', '1e-3', '10UL', '.5d', '2.5E+10']) { + expect(classes(`var n = ${number};`, number), number).toEqual(['number']) + } + expect(classes('var v = 1.2.3; var id = item1;', '1.2.3')).toEqual([undefined]) + expect(classes('var v = 1.2.3; var id = item1;', 'item1')).toEqual([undefined]) + const code = 'var s = items[1..^1]; var t = x?[0] ?? y!; p->Next; global::System.Math.Max(a, b); f = x => x;' + expect(classes(code, '1..^1')).toEqual(['number', 'operator', 'operator', 'number']) + expect(classes(code, 'x?[0] ?? y!')).toEqual([undefined, 'operator', undefined, 'number', undefined, 'operator', undefined, 'operator']) + expect(classes(code, 'p->Next')).toEqual([undefined, 'operator', 'type']) + expect(classes(code, 'global::System.Math.Max(')).toEqual(['keyword', 'operator', 'type', undefined, 'property', undefined, 'function', undefined]) + expect(classes(code, 'x => x')).toEqual([undefined, 'operator', undefined]) + expect(classes('var r = items[start..end];', 'start..end')).toEqual([undefined, 'operator', undefined]) + expect(classes('var p = Get>(key);', 'Get { + const code = `using static System.Math;\nusing Json = System.Text.Json.JsonSerializer;\napp.MapPost("/u", async ([FromBody] User dto, [FromServices] AppDb db) => 1);\nforeach (var file in files) { if (file is null) return; }\nforeach (var group in groups) { }\npublic readonly record struct Point(int X, int Y);\npublic string FullName => $"{First} {Last}";\nvar k = s switch { Circle => "c", _ => "?" };` + expect(classes(code, 'static System.Math;')).toEqual(['keyword', undefined, 'type', undefined]) + expect(classes(code, 'Json = System.Text.Json.JsonSerializer;')).toEqual(['type', undefined, 'operator', undefined, 'type', undefined]) + expect(classes(code, '[FromBody] User')).toEqual([undefined, 'attr', undefined, 'type']) + expect(classes(code, '[FromServices]')).toEqual([undefined, 'attr', undefined]) + expect(classes(code, 'var file in files')).toEqual(['keyword', undefined, 'keyword', undefined]) + expect(classes(code, 'file is null')).toEqual([undefined, 'keyword', undefined, 'literal']) + expect(classes(code, 'var group in')).toEqual(['keyword', undefined, 'keyword']) + expect(classes('var @class = 1;', '@class')).toEqual([undefined]) + expect(classes(code, 'record struct Point(')).toEqual(['keyword', undefined, 'keyword', undefined, 'type', undefined]) + expect(classes(code, 'string FullName =>')).toEqual(['type', undefined, 'property', undefined, 'operator']) + expect(classes(code, 'Circle =>')).toEqual(['type', undefined, 'operator']) + const strings = `Foo("", "a"); var o = x == "" ? "a" : "b"; var v = "a""b";\nvar u = $"{(c == '"' ? 1 : 2)}"; var w = $"{new { A = 1 }.A} // {"}"}"; var z = 1;\nvar s = @"one\n#if NOT_META\n// not comment\n"; var r = """\n /* no */ ""two""\n """; var after = 2;` + expect(classes(strings, 'Foo("", "a")')).toEqual(['function', undefined, 'string', undefined, 'string', undefined]) + expect(classes(strings, 'x == "" ? "a" : "b"')).toEqual([undefined, 'operator', undefined, 'string', undefined, 'operator', undefined, 'string', undefined, 'operator', undefined, 'string']) + expect(classes(strings, '"a""b"')).toEqual(['string', 'string']) + expect(classes(strings, `$"{(c == '"' ? 1 : 2)}"`)).toEqual(['string']) + expect(classes(strings, '$"{new { A = 1 }.A} // {"}"}"')).toEqual(['string']) + expect(classes(strings, 'var z = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes(strings, '@"one\n#if NOT_META\n// not comment\n"')).toEqual(['string']) + expect(classes(strings, '"""\n /* no */ ""two""\n """')).toEqual(['string']) + expect(classes(strings, 'var after')).toEqual(['keyword', undefined]) + }) + + it('registers aliases', () => { + for (const name of ['csharp', 'c#', 'cs']) { + expect(highlighter.normalizeLanguage(name)).toBe('csharp') + } + expect(createHighlighter({ languages: [] }).normalizeLanguage('cs')).toBe('plaintext') + }) +}) diff --git a/test/dart.test.ts b/test/dart.test.ts new file mode 100644 index 0000000..9723d61 --- /dev/null +++ b/test/dart.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { dart } from '../src/languages/dart' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [dart] }) +const _ = undefined +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'dart') + +describe('Dart documentation syntax', () => { + it('highlights a realistic Flutter widget', () => { + const code = `import 'package:flutter/material.dart'; + +/// A counter page. Don't forget the "key". +@immutable +class CounterPage extends StatefulWidget { + const CounterPage({super.key, required this.title}); + + final String title; + + @override + State createState() => _CounterPageState(); +} + +class _CounterPageState extends State { + int _count = 0; + final Map> cache = {'k': [1_000, 0xFF]}; + String? name; + + void _increment() => setState(() => _count++); + + Stream ticks() async* { + for (var i = 0; i < 10; i++) { + yield* Stream.value(i); + } + } + + @override + Widget build(BuildContext context) { + final label = 'Hello $name, \${cache['k']?.first ?? 0}'; + return Scaffold( + appBar: AppBar(title: Text(widget.title)), + body: Padding( + padding: const EdgeInsets.all(8), + child: Text(label, style: Theme.of(context).textTheme.headlineMedium), + ), + floatingActionButton: FloatingActionButton( + onPressed: _increment, + child: const Icon(Icons.add), + ), + ); + } +} +` + expect(classes(code, "'package:flutter/material.dart'")).toEqual(['string']) + expect(classes(code, '/// A counter page.')).toEqual(['comment']) + expect(classes(code, '@immutable')).toEqual(['attr']) + expect(classes(code, 'CounterPage extends')).toEqual(['type', _, 'keyword']) + expect(classes(code, 'super.key')).toEqual(['keyword', _, 'property']) + expect(classes(code, 'required this')).toEqual(['keyword', _, 'keyword']) + expect(classes(code, 'createState()')).toEqual(['function', _]) + expect(classes(code, '_CounterPageState extends')).toEqual(['type', _, 'keyword']) + expect(classes(code, 'Map>')).toEqual(['type', 'operator', 'type', _, 'type', 'operator', 'type', 'operator']) + expect(classes(code, '1_000, 0xFF')).toEqual(['number', _, 'number']) + expect(classes(code, 'String? name')).toEqual(['type', 'operator', _]) + expect(classes(code, 'async*')).toEqual(['keyword']) + expect(classes(code, 'yield* Stream.value')).toEqual(['keyword', _, 'type', _, 'function']) + expect(classes(code, "'Hello $name, ${cache['k']?.first ?? 0}'")).toEqual(['string']) + expect(classes(code, 'appBar: AppBar(title: Text(widget.title))')).toEqual([ + 'property', 'operator', _, 'type', _, 'property', 'operator', _, 'type', _, 'property', _, + ]) + expect(classes(code, 'const EdgeInsets.all(8)')).toEqual(['keyword', _, 'type', _, 'function', _, 'number', _]) + expect(classes(code, 'Theme.of(context).textTheme')).toEqual(['type', _, 'function', _, 'property']) + }) + + it('finds the end of interpolated strings with nested quotes and braces', () => { + const nested = `'\${map['key']} and \${items.map((e) => "<$e>").join()} it\\'s {}'` + const deep = `"\${'a \${"b \${'c'}"}'} // not a comment"` + const code = `final a = ${nested}; // tail\nfinal b = ${deep};\nfinal c = "\${{'k': 1}['k']}";` + expect(classes(code, nested)).toEqual(['string']) + expect(classes(code, '// tail')).toEqual(['comment']) + expect(classes(code, deep)).toEqual(['string']) + expect(classes(code, `"\${{'k': 1}['k']}"`)).toEqual(['string']) + expect(classes(code, 'final c')).toEqual(['keyword', _]) + }) + + it('handles triple-quoted, raw, and adjacent strings', () => { + const triple = `'''\nIt's a "test" with \${user['name']}\nand ''\n'''` + const doubleTriple = `"""\nShe said "hi" \${"x"}\n"""` + const raw = `r'C:\\path$notInterp'` + const rawDouble = `r"C:\\dir\\"` + const rawTriple = `r'''$a \${'''` + const code = `var t = ${triple};\nvar u = ${doubleTriple};\nvar p = ${raw}; var q = ${rawDouble}; var s = ${rawTriple};\nvar j = 'a' "b" 'c';\nvar bar = 1;` + expect(classes(code, triple)).toEqual(['string']) + expect(classes(code, doubleTriple)).toEqual(['string']) + expect(classes(code, raw)).toEqual(['string']) + expect(classes(code, rawDouble)).toEqual(['string']) + expect(classes(code, `${rawTriple};`)).toEqual(['string', _]) + expect(classes(code, `'a' "b" 'c'`)).toEqual(['string', _, 'string', _, 'string']) + expect(classes(code, 'var bar')).toEqual(['keyword', _]) + }) + + it('resets raw-ness and escapes around nested and dollar-heavy strings', () => { + const code = `var a = "\${r'raw\\'} \\\${x} \${'}'}" + '\${names[r]}' + 'cost: \\$\${price}' + '$';\nvar j = r'\\'; var k = 1;\nfor (final r in ['a']) {}` + expect(classes(code, `"\${r'raw\\'} \\\${x} \${'}'}"`)).toEqual(['string']) + expect(classes(code, `'\${names[r]}' + 'cost: \\$\${price}' + '$'`)).toEqual(['string', _, 'operator', _, 'string', _, 'operator', _, 'string']) + expect(classes(code, `r'\\'; var k`)).toEqual(['string', _, 'keyword', _]) + expect(classes(code, "r in ['a']")).toEqual([_, 'keyword', _, 'string', _]) + }) + + it('keeps unterminated single-line strings to their line', () => { + const code = "var s = 'oops\nvar n = 1;" + expect(classes(code, "'oops")).toEqual(['string']) + expect(classes(code, 'var n')).toEqual(['keyword', _]) + }) + + it('nests block comments and isolates delimiters inside comments', () => { + const block = `/* outer /* inner "x" */ still 'comment' */` + const code = `${block} int after;\n// don't "stop"\nvar x = 1; /// doc's\nvar y = '/* not a comment */ // nor this';` + expect(classes(code, block)).toEqual(['comment']) + expect(classes(code, 'int after')).toEqual(['type', _]) + expect(classes(code, `// don't "stop"`)).toEqual(['comment']) + expect(classes(code, 'var x')).toEqual(['keyword', _]) + expect(classes(code, "/// doc's")).toEqual(['comment']) + expect(classes(code, "'/* not a comment */ // nor this'")).toEqual(['string']) + }) + + it('classifies annotations, keywords, literals, and types', () => { + const code = `@Deprecated('use x')\n@pragma('vm:entry-point')\nlate final bool? ok = true;\nvoid Function(int) cb;\nNever fail() => throw null;\nextension StringX on String {}\nenum color { red }\nfinal ok = x is! Foo;\nconst MAX_SIZE = 1;\nconst Foo({required this.count}) : assert(count >= 0);\nvoid check() { assert(x != null, 'msg'); }` + expect(classes(code, "@Deprecated('use x')")).toEqual(['attr', _, 'string', _]) + expect(classes(code, "@pragma('vm:entry-point')")).toEqual(['attr', _, 'string', _]) + expect(classes(code, 'late final bool? ok = true')).toEqual(['keyword', _, 'keyword', _, 'type', 'operator', _, 'operator', _, 'literal']) + expect(classes(code, 'void Function(int)')).toEqual(['type', _, 'type', _, 'type', _]) + expect(classes(code, 'Never fail() => throw null')).toEqual(['type', _, 'function', _, 'operator', _, 'keyword', _, 'literal']) + expect(classes(code, 'extension StringX on String')).toEqual(['keyword', _, 'type', _, 'keyword', _, 'type']) + expect(classes(code, 'enum color')).toEqual(['keyword', _, 'type']) + expect(classes(code, 'is! Foo')).toEqual(['keyword', 'operator', _, 'type']) + expect(classes(code, 'MAX_SIZE')).toEqual([_]) + expect(classes(code, ': assert(count >= 0)')).toEqual(['operator', _, 'keyword', _, 'operator', _, 'number', _]) + expect(classes(code, "assert(x != null, 'msg')")).toEqual(['keyword', _, 'operator', _, 'literal', _, 'string', _]) + }) + + it('separates numbers from cascades, spreads, and member access', () => { + for (const number of ['1_000', '1__000', '0xFF', '0xFF_FF', '1.5e3', '1e-3', '.5', '42']) { + expect(classes(`var n = ${number};`, number), number).toEqual(['number']) + } + const code = 'final l = [...items, ...?other]..add(1)..length = 2;\nobj?..foo();\nfinal s = 1.toString() + v1.2.3 + 1.2.3;' + expect(classes(code, '...items')).toEqual(['operator', _]) + expect(classes(code, '...?other')).toEqual(['operator', _]) + expect(classes(code, '..add(1)')).toEqual(['operator', 'function', _, 'number', _]) + expect(classes(code, '..length')).toEqual(['operator', 'property']) + expect(classes(code, '?..foo')).toEqual(['operator', 'function']) + expect(classes(code, '1.toString')).toEqual(['number', _, 'function']) + expect(classes(code, '+ 1.2.3')).toEqual(['operator', _]) + }) + + it('labels named arguments and records but not ternaries', () => { + const code = "foo(a ? b : c, flag ? x: y);\nfinal r = (name: 'x', age: 3);\nfinal m = {'k': v};\nswitch (shape) { Circle(:final radius) when radius > 0 => radius, _ => 0 }\nx ??= y ?? z ~/ 2;" + expect(classes(code, 'a ? b : c')).toEqual([_, 'operator', _, 'operator', _]) + expect(classes(code, 'x: y')).toEqual([_, 'operator', _]) + expect(classes(code, "(name: 'x', age: 3)")).toEqual([_, 'property', 'operator', _, 'string', _, 'property', 'operator', _, 'number', _]) + expect(classes(code, "{'k': v}")).toEqual([_, 'string', 'operator', _]) + expect(classes(code, 'Circle(:final radius) when')).toEqual(['type', _, 'operator', 'keyword', _, 'keyword']) + expect(classes(code, 'x ??= y ?? z ~/ 2')).toEqual([_, 'operator', _, 'operator', _, 'operator', _, 'number']) + }) + + it('colours contextual keywords only where they are used as keywords', () => { + const code = `import 'a.dart' deferred as a show Foo hide Bar;\npart 'x.g.dart';\nbase mixin M on Object {}\nsealed class Store extends _$Store {}\nextension type Id(int v) {}\nint get value => _v;\nset value(int v) {}\nlate final List xs;\nfinal set = {}, base = 10;\nfor (final part in parts) {}\nif (when == null) show();\nfoo(show: true, on: false);\non((e) => e);\nswitch (x) {\n default:\n}` + expect(classes(code, 'deferred as a show Foo hide Bar')).toEqual(['keyword', _, 'keyword', _, 'keyword', _, 'type', _, 'keyword', _, 'type']) + expect(classes(code, "part 'x.g.dart'")).toEqual(['keyword', _, 'string']) + expect(classes(code, 'base mixin M on Object')).toEqual(['keyword', _, 'keyword', _, 'type', _, 'keyword', _, 'type']) + expect(classes(code, 'sealed class Store extends _$Store')).toEqual(['keyword', _, 'keyword', _, 'type', _, 'keyword', _, 'type']) + expect(classes(code, 'extension type Id(int')).toEqual(['keyword', _, 'type', _, 'type']) + expect(classes(code, 'int get value')).toEqual(['type', _, 'keyword', _]) + expect(classes(code, 'set value(')).toEqual(['keyword', _, 'function', _]) + expect(classes(code, 'late final')).toEqual(['keyword', _, 'keyword']) + expect(classes(code, 'set = {}, base =')).toEqual([_, 'operator', _, 'operator', 'type', 'operator', _, 'operator']) + expect(classes(code, 'final part in')).toEqual(['keyword', _, 'keyword']) + expect(classes(code, 'when == null) show()')).toEqual([_, 'operator', _, 'literal', _, 'function', _]) + expect(classes(code, 'show: true, on: false')).toEqual(['property', 'operator', _, 'literal', _, 'property', 'operator', _, 'literal']) + expect(classes(code, 'on')).toEqual([_, 'operator', 'type', 'operator']) + expect(classes(code, 'default:')).toEqual(['keyword', 'operator']) + }) + + it('keeps keywords after member access plain', () => { + const code = 'value.when(data: (d) => d);\nfinal s = obj.set;' + expect(classes(code, 'when(data:')).toEqual(['function', _, 'property', 'operator']) + expect(classes(code, 'obj.set')).toEqual([_, 'property']) + }) + + it('registers only the dart name', () => { + expect(highlighter.normalizeLanguage('dart')).toBe('dart') + expect(createHighlighter({ languages: [] }).normalizeLanguage('dart')).toBe('plaintext') + }) +}) diff --git a/test/fixtures.ts b/test/fixtures.ts index f4ef90a..94ad8dd 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -26,6 +26,54 @@ export const languageFixtures: Array = [ code: 'cmake_minimum_required(VERSION 3.20)\nproject(Hello)\nset(ENABLED ON)\nif(ENABLED)\n message("hello")\n add_subdirectory(${SOURCE_DIR})\nendif() # done', expectedClasses: ['th-command', 'th-number', 'th-literal', 'th-keyword', 'th-string', 'th-variable', 'th-comment'], }, + { + lang: 'c#', + normalized: 'csharp', + code: 'using System;\n[Serializable]\npublic record Point(int X, int Y);\nvar name = "world"; // greet\nConsole.WriteLine($"Hello {name}", 42);', + expectedClasses: ['th-keyword', 'th-type', 'th-string', 'th-comment', 'th-function', 'th-number', 'th-operator'], + }, + { + lang: 'dart', + normalized: 'dart', + code: `import 'package:flutter/material.dart';\n@override\nWidget build(BuildContext context) {\n final count = 42; // total\n return Text('Hello $name', key: null);\n}`, + expectedClasses: ['th-keyword', 'th-string', 'th-type', 'th-function', 'th-number', 'th-operator', 'th-comment'], + }, + { + lang: 'java', + normalized: 'java', + code: 'import java.util.List;\n@Override\npublic String greet(String name) {\n int count = 42; // total\n return "Hello " + name;\n}', + expectedClasses: ['th-keyword', 'th-type', 'th-function', 'th-number', 'th-operator', 'th-comment', 'th-string'], + }, + { + lang: 'kt', + normalized: 'kotlin', + code: 'import kotlin.math.max\n@JvmStatic\nfun greet(name: String): String {\n val count = 42 // total\n return "Hello ${name}"\n}', + expectedClasses: ['th-keyword', 'th-function', 'th-type', 'th-number', 'th-operator', 'th-comment', 'th-string'], + }, + { + lang: 'lua', + normalized: 'lua', + code: 'local function greet(name)\n -- say hello\n local count = 42\n return "Hello " .. name, nil\nend\nprint(greet("world"))', + expectedClasses: ['th-keyword', 'th-function', 'th-comment', 'th-number', 'th-operator', 'th-string', 'th-literal'], + }, + { + lang: 'pl', + normalized: 'perl', + code: 'use strict;\nmy $name = "world";\nmy @items = (1, 2, 42); # list\nsub greet { return "Hello $_[0]"; }\nprint greet($name) if $name =~ /wor/;', + expectedClasses: ['th-keyword', 'th-variable', 'th-operator', 'th-string', 'th-number', 'th-comment', 'th-function'], + }, + { + lang: 'rb', + normalized: 'ruby', + code: `require 'json'\nclass Greeter\n def greet(name) # say hello\n "Hello #{name}" * 42\n end\nend`, + expectedClasses: ['th-keyword', 'th-string', 'th-type', 'th-function', 'th-comment', 'th-operator', 'th-number'], + }, + { + lang: 'rs', + normalized: 'rust', + code: 'use std::fmt;\n#[derive(Debug)]\nstruct Point { x: i32 }\nfn main() {\n let count = 42; // total\n println!("Hello {}", count);\n}', + expectedClasses: ['th-keyword', 'th-type', 'th-function', 'th-number', 'th-operator', 'th-comment', 'th-string'], + }, { lang: 'octane', normalized: 'tsrx', diff --git a/test/java.test.ts b/test/java.test.ts new file mode 100644 index 0000000..f0ae609 --- /dev/null +++ b/test/java.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { java } from '../src/languages/java' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [java] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'java') + +const showcase = `package com.example.cache; + +import java.time.Duration; +import java.util.concurrent.ConcurrentHashMap; +import static java.util.Objects.requireNonNull; + +/** + * A tiny TTL cache. Use {@code get("key")} to read; don't share across tenants. + */ +public final class TtlCache implements AutoCloseable { + private static final long DEFAULT_TTL_MS = 60_000L; + private final Map> entries = new ConcurrentHashMap<>(); + private final Duration ttl; + + public sealed interface Result permits Hit, Miss {} + public record Hit(V value, long age) implements Result {} + public record Miss() implements Result {} + + public TtlCache(Duration ttl) { + this.ttl = requireNonNull(ttl, "ttl must not be null"); + } + + @Override + @SuppressWarnings("unchecked") + public void close() { + entries.clear(); // drop everything + } + + public Result get(K key) { + var entry = entries.get(key); + if (entry == null || entry.isExpired(ttl)) return new Miss<>(); + return new Hit<>(entry.value(), System.nanoTime() - entry.createdAt()); + } + + public String describe(Result result) { + return switch (result) { + case Hit hit when hit.age() > 0x1F -> "hit after " + hit.age() + "ns"; + case Miss miss -> { + String help = """ + Nothing cached for "%s". + See https://example.com/docs // not a comment + """; + yield help.formatted(miss); + } + }; + } + + public void dump() { + entries.keySet().stream().map(String::valueOf).forEach(System.out::println); + } +} +` + +describe('Java documentation syntax', () => { + it('highlights a realistic class', () => { + expect(classes(showcase, 'package')).toEqual(['keyword']) + expect(classes(showcase, 'java.time.Duration')).toEqual([undefined]) + expect(classes(showcase, '/**')).toEqual(['comment']) + expect(classes(showcase, '{@code get("key")}')).toEqual(['comment']) + expect(classes(showcase, 'TtlCache')).toEqual(['type']) + expect(classes(showcase, 'implements')).toEqual(['keyword']) + expect(classes(showcase, 'DEFAULT_TTL_MS')).toEqual([undefined]) + expect(classes(showcase, '60_000L')).toEqual(['number']) + expect(classes(showcase, 'sealed')).toEqual(['keyword']) + expect(classes(showcase, 'permits')).toEqual(['keyword']) + expect(classes(showcase, 'record')).toEqual(['keyword']) + expect(classes(showcase, 'this.ttl')).toEqual(['keyword', undefined, 'property']) + expect(classes(showcase, 'requireNonNull(ttl')).toEqual(['function', undefined]) + expect(classes(showcase, '"ttl must not be null"')).toEqual(['string']) + expect(classes(showcase, '@Override')).toEqual(['attr']) + expect(classes(showcase, '@SuppressWarnings("unchecked")')).toEqual(['attr', undefined, 'string', undefined]) + expect(classes(showcase, 'void')).toEqual(['type']) + expect(classes(showcase, '// drop everything')).toEqual(['comment']) + expect(classes(showcase, 'var entry')).toEqual(['keyword', undefined]) + expect(classes(showcase, 'null ||')).toEqual(['literal', undefined, 'operator']) + expect(classes(showcase, 'hit when hit')).toEqual([undefined, 'keyword', undefined]) + expect(classes(showcase, '0x1F')).toEqual(['number']) + expect(classes(showcase, '-> "hit')).toEqual(['operator', undefined, 'string']) + expect(classes(showcase, '"""\n Nothing')).toEqual(['string']) + expect(classes(showcase, 'yield')).toEqual(['keyword']) + expect(classes(showcase, 'String::valueOf')).toEqual(['type', 'operator', 'function']) + expect(classes(showcase, 'System.out::println')).toEqual(['type', undefined, 'property', 'operator', 'function']) + }) + + it('keeps text blocks whole, including quotes, escapes, and comment markers', () => { + const block = '"""\n {"a": "b", "c": ""} // still text\n \\""" /* also text */\n """' + const code = `String json = ${block};\nint after = 1;` + expect(classes(code, block)).toEqual(['string']) + expect(classes(code, 'int after')).toEqual(['type', undefined]) + expect(classes(code, '1;')).toEqual(['number', undefined]) + const empty = 'String s = """\n""";' + expect(classes(empty, '"""\n"""')).toEqual(['string']) + }) + + it('handles ordinary strings and char literals', () => { + const code = 'String url = "http://x.com/*y*/";\nString q = "say \\"hi\\" // no";\nchar a = \'\\\'\', b = \'"\', c = \'A\', d = \'\\u0041\';\nString e = "it\'s";' + expect(classes(code, '"http://x.com/*y*/"')).toEqual(['string']) + expect(classes(code, '"say \\"hi\\" // no"')).toEqual(['string']) + expect(classes(code, "'\\''")).toEqual(['string']) + expect(classes(code, '\'"\'')).toEqual(['string']) + expect(classes(code, "'A'")).toEqual(['string']) + expect(classes(code, "'\\u0041'")).toEqual(['string']) + expect(classes(code, '"it\'s"')).toEqual(['string']) + }) + + it('isolates line, block, and Javadoc comments', () => { + const code = '// don\'t "quote"\nint x; /* it\'s "fine" */ int y;\n/** @param z the "z" */\nvoid f(int z) {}' + expect(classes(code, '// don\'t "quote"')).toEqual(['comment']) + expect(classes(code, '/* it\'s "fine" */')).toEqual(['comment']) + expect(classes(code, 'int y')).toEqual(['type', undefined]) + expect(classes(code, '/** @param z the "z" */')).toEqual(['comment']) + expect(classes(code, 'f(')).toEqual(['function', undefined]) + }) + + it('stops unterminated strings at end of line and runs text blocks and comments to EOF', () => { + const code = 'String s = "open\nint next = 2;' + expect(classes(code, '"open')).toEqual(['string']) + expect(classes(code, 'int next')).toEqual(['type', undefined]) + const slash = 'String s = "abc\\\nint y = 1;' + expect(classes(slash, '"abc\\')).toEqual(['string']) + expect(classes(slash, 'int y')).toEqual(['type', undefined]) + expect(classes('int a; /* open\n "x" int b;', '/* open\n "x" int b;')).toEqual(['comment']) + expect(classes('String t = """\n open \\', '"""\n open \\')).toEqual(['string']) + expect(classes('String s = "x\\', '"x\\')).toEqual(['string']) + }) + + it('distinguishes annotations from @interface declarations', () => { + const code = '@interface Marker { String value() default ""; }\n@java.lang.Deprecated(since = "9")\n@Marker class Foo {}' + expect(classes(code, '@interface')).toEqual(['keyword']) + expect(classes(code, 'Marker {')).toEqual(['type', undefined]) + expect(classes(code, 'default')).toEqual(['keyword']) + expect(classes(code, '@java.lang.Deprecated')).toEqual(['attr']) + expect(classes(code, '@Marker')).toEqual(['attr']) + }) + + it('recognizes Java numeric literal forms', () => { + for (const number of ['1_000_000', '0xFF_FFL', '0b1010', '017', '1.5f', '1e-3d', '100L', '0x1.8p1', '.5', '3.', '1E10F', '0B1_0']) { + const code = `double v = ${number};` + expect(classes(code, number), number).toEqual(['number']) + } + const code = 'int x1 = a.b + v1.2.3;' + expect(classes(code, 'x1')).toEqual([undefined]) + expect(classes(code, '1.2.3')).toEqual([undefined]) + }) + + it('types PascalCase names and primitives but not constants', () => { + const code = 'List> rows; boolean ok; T item; MAX_SIZE; Foo.class; int[] xs;' + expect(classes(code, 'List>')).toEqual(['type', 'operator', 'type', 'operator', 'type', undefined, 'type', 'operator']) + expect(classes(code, 'boolean')).toEqual(['type']) + expect(classes(code, 'T item')).toEqual([undefined]) + expect(classes(code, 'MAX_SIZE')).toEqual([undefined]) + expect(classes(code, 'Foo.class')).toEqual(['type', undefined, 'keyword']) + const varargs = 'static void main(String... args) { int.class; }' + expect(classes(varargs, 'String... args)')).toEqual(['type', undefined]) + expect(classes(varargs, 'int.class')).toEqual(['type', undefined, 'keyword']) + }) + + it('treats modern contextual keywords only in keyword position', () => { + const code = 'non-sealed class A {}\nrecord Point(int x, int y) {}\nvar list = record.items();\nint var = 1;\nObject o = obj instanceof String s ? s : null;\nRunnable r = () -> super.run();\nint when = 1;\nfoo(when);\nthis.when = when.toString();' + expect(classes(code, 'non-sealed')).toEqual(['keyword']) + expect(classes(code, 'record Point')).toEqual(['keyword', undefined, 'type']) + expect(classes(code, 'var list')).toEqual(['keyword', undefined]) + expect(classes(code, 'record.items')).toEqual([undefined, 'function']) + expect(classes(code, 'var = 1')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, 'instanceof String s')).toEqual(['keyword', undefined, 'type', undefined]) + expect(classes(code, '() -> super')).toEqual([undefined, 'operator', undefined, 'keyword']) + expect(classes(code, 'int when = 1')).toEqual(['type', undefined, 'operator', undefined, 'number']) + expect(classes(code, 'foo(when)')).toEqual(['function', undefined]) + expect(classes(code, 'this.when = when.toString()')).toEqual(['keyword', undefined, 'property', undefined, 'operator', undefined, 'function', undefined]) + }) + + it('handles module declarations and import paths', () => { + const code = 'module com.example.app {\n requires transitive java.base;\n exports com.example.api;\n}\nimport java.util.*;\n// import not.a.Path;\nString s = "import a.b";' + expect(classes(code, 'module')).toEqual(['keyword']) + expect(classes(code, 'requires transitive')).toEqual(['keyword']) + expect(classes(code, 'exports')).toEqual(['keyword']) + expect(classes(code, 'import java.util.*;')).toEqual(['keyword', undefined]) + expect(classes(code, '// import not.a.Path;')).toEqual(['comment']) + expect(classes(code, '"import a.b"')).toEqual(['string']) + }) + + it('registers no aliases and falls back to plaintext when absent', () => { + expect(highlighter.normalizeLanguage('java')).toBe('java') + expect(createHighlighter({ languages: [] }).normalizeLanguage('java')).toBe('plaintext') + }) +}) diff --git a/test/kotlin.test.ts b/test/kotlin.test.ts new file mode 100644 index 0000000..caf08cf --- /dev/null +++ b/test/kotlin.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { kotlin } from '../src/languages/kotlin' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [kotlin] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'kotlin') + +const showcase = `@file:JvmName("Users") +package com.example.users + +import kotlinx.coroutines.flow.Flow + +/** + * Loads users. Nested /* "comments" */ are fine. + */ +@Serializable +data class User(val id: Long, val name: String, val email: String? = null) + +sealed interface Result { + data class Ok(val value: T) : Result + data object Missing : Result +} + +class UserRepository private constructor(private val api: Api) { + companion object { + @JvmStatic fun create(api: Api) = UserRepository(api) + const val MAX_SIZE = 1_000 + } + + var loaded: Int = 0 + private set + + val isEmpty: Boolean + get() = loaded == 0 + + suspend fun find(id: Long): Result { + val user = api.fetch(id)?.takeIf { it.id > 0L } ?: return Result.Missing + loaded++ + println("Loaded \${user.name} (\$id) // cached") + return Result.Ok(user) + } +} + +fun String.shout() = uppercase() + "!" + +fun main() = runBlocking { + val repo = UserRepository.create(HttpApi()) + when (val result = repo.find(42L)) { + is Result.Ok -> println(result.value.name.shout()) + Result.Missing -> println("missing") + } + for (i in 0..<3) if (i !in setOf(1, 2)) println('#') +} +` + +describe('Kotlin documentation syntax', () => { + it('highlights a realistic showcase sample', () => { + const code = showcase + expect(classes(code, '@file:JvmName')).toEqual(['attr']) + expect(classes(code, '"Users"')).toEqual(['string']) + expect(classes(code, 'com.example.users')).toEqual([undefined]) + expect(classes(code, '/**')).toEqual(['comment']) + expect(classes(code, 'are fine.\n */')).toEqual(['comment']) + expect(classes(code, '@Serializable')).toEqual(['attr']) + expect(classes(code, 'data class User')).toEqual(['keyword', undefined, 'keyword', undefined, 'type']) + expect(classes(code, 'String? = null')).toEqual(['type', undefined, 'operator', undefined, 'literal']) + expect(classes(code, 'out T')).toEqual(['keyword', undefined]) + expect(classes(code, 'data object')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'private constructor')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'companion object')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, '@JvmStatic fun create')).toEqual(['attr', undefined, 'keyword', undefined, 'function']) + expect(classes(code, 'MAX_SIZE = 1_000')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, 'private set')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'get() =')).toEqual(['keyword', undefined, 'operator']) + expect(classes(code, 'suspend fun find')).toEqual(['keyword', undefined, 'keyword', undefined, 'function']) + expect(classes(code, 'api.fetch(id)?.takeIf {')).toEqual([undefined, 'function', undefined, 'operator', 'function', undefined]) + expect(classes(code, '0L } ?: return')).toEqual(['number', undefined, 'operator', undefined, 'keyword']) + expect(classes(code, 'loaded++')).toEqual([undefined, 'operator']) + expect(classes(code, '"Loaded ${user.name} ($id) // cached"')).toEqual(['string']) + expect(classes(code, 'String.shout')).toEqual(['type', undefined, 'function']) + expect(classes(code, 'runBlocking {')).toEqual(['function', undefined]) + expect(classes(code, 'HttpApi()')).toEqual(['function', undefined]) + expect(classes(code, 'Result.Ok(user)')).toEqual(['type', undefined, 'function', undefined]) + expect(classes(code, 'is Result.Ok ->')).toEqual(['keyword', undefined, 'type', undefined, 'type', undefined, 'operator']) + expect(classes(code, 'result.value.name')).toEqual([undefined, 'property', undefined, 'property']) + expect(classes(code, '0..<3')).toEqual(['number', 'operator', 'number']) + expect(classes(code, '!in')).toEqual(['keyword']) + expect(classes(code, "'#'")).toEqual(['string']) + }) + + it('finds the end of templates with nested quotes and braces', () => { + for (const string of [ + '"Hello, $name"', + '"${user.name}"', + '"value: ${map["key"]} // not a comment"', + '"${if (a) "x" else "y"}"', + '"${list.map { it.name }}"', + '"${"}"}"', + '"${listOf("a", "b").joinToString { "[$it]" }}"', + '"price: \\$5 and \\"quoted\\" /* no */"', + '"char ${\'"\'} and ${\'\\\'\'}"', + '"${\'$\'}9.99 ${a}${b} } { cost$"', + ]) { + const code = `val text = ${string} + 1 // tail` + expect(classes(code, string), string).toEqual(['string']) + expect(classes(code, '1'), string).toEqual(['number']) + expect(classes(code, '// tail'), string).toEqual(['comment']) + } + }) + + it('scans raw strings with quotes, backslashes, templates, and trailing quotes', () => { + const raw = '"""\n {"name": "${user.name}", "path": "C:\\temp\\"} // "" not a comment\n ${"\\"\\"\\""} /* nor this */\n"""' + const trailing = '"""ends with a quote""""' + const dollar = '$$"""{"$schema": "$${id}"}"""' + const code = `val json = ${raw}.trimIndent()\nval quoted = ${trailing}\nval schema = ${dollar}\nval done = true` + expect(classes(code, raw)).toEqual(['string']) + expect(classes(code, '.trimIndent')).toEqual([undefined, 'function']) + expect(classes(code, trailing)).toEqual(['string']) + expect(classes(code, dollar)).toEqual(['string']) + expect(classes(code, 'true')).toEqual(['literal']) + }) + + it('handles char literals, nested comments, and unterminated strings', () => { + const code = [ + "val chars = listOf('a', '\\n', '\\'', '\"', '/', '\\u0041')", + '/* outer /* inner "quote */ it\'s still */ val after = 1', + '/** KDoc with `code` and "quotes" */', + '// line comment with "quote and \'char', + 'val broken = "no end', + 'val next = 2', + ].join('\n') + for (const char of ["'a'", "'\\n'", "'\\''", "'\"'", "'/'", "'\\u0041'"]) { + expect(classes(code, char), char).toEqual(['string']) + } + expect(classes(code, '/* outer /* inner "quote */ it\'s still */')).toEqual(['comment']) + expect(classes(code, 'val after = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes(code, '/** KDoc with `code` and "quotes" */')).toEqual(['comment']) + expect(classes(code, '// line comment with "quote and \'char')).toEqual(['comment']) + expect(classes(code, '"no end')).toEqual(['string']) + expect(classes(code, 'val next = 2')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + }) + + it('separates annotations from labels and keeps backticked names intact', () => { + const code = [ + '@get:Rule val rule = TestRule()', + '@Suppress("UNCHECKED_CAST") class Box', + 'outer@ for (x in xs) {', + ' xs.forEach { if (it == x) return@forEach else break@outer }', + ' this@Box.size', + '}', + '@Test fun `does a "thing" when null`() { val v = `class` }', + ].join('\n') + expect(classes(code, '@get:Rule')).toEqual(['attr']) + expect(classes(code, '@Suppress')).toEqual(['attr']) + expect(classes(code, 'outer@ for')).toEqual([undefined, 'keyword']) + expect(classes(code, 'return@forEach')).toEqual(['keyword', undefined]) + expect(classes(code, 'break@outer')).toEqual(['keyword', undefined]) + expect(classes(code, 'this@Box')).toEqual(['keyword', undefined, 'type']) + expect(classes(code, '`does a "thing" when null`')).toEqual(['function']) + expect(classes(code, '`class`')).toEqual([undefined]) + }) + + it('treats soft keywords by context', () => { + const code = [ + 'value class Id(val raw: Int)', + 'enum class Color { RED, GREEN }', + 'open annotation class Marker', + 'inline fun List<*>.only(vararg items: T) = filterIsInstance()', + 'val data = map.get(0) + value + open', + 'val result = data', + 'val x = y as? String ?: z!!', + 'expect fun platform(): String', + 'var count = 0', + ' private set', + ' set(value) { field = value }', + 'val p: Int get() { return 1 }', + 'val s = set', + 'routing { get("/users/{id}") { }; get(path) { }; get { } }', + 'list.set(0, x)', + '@file:JvmName("X")', + '@OptIn(ExperimentalFoo::class) fun f(@PathVariable id: Long) = "user@example.com"', + ].join('\n') + expect(classes(code, 'private set')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'set(value) {')).toEqual(['keyword', undefined]) + expect(classes(code, 'get() {')).toEqual(['keyword', undefined]) + expect(classes(code, 's = set')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'get("/users/{id}") { }; get(path) { }; get {')).toEqual(['function', undefined, 'string', undefined, 'function', undefined, 'function', undefined]) + expect(classes(code, 'list.set(')).toEqual([undefined, 'function', undefined]) + expect(classes(code, '@file:JvmName("X")')).toEqual(['attr', undefined, 'string', undefined]) + expect(classes(code, '@OptIn(ExperimentalFoo::class)')).toEqual(['attr', undefined, 'type', 'operator', 'keyword', undefined]) + expect(classes(code, '@PathVariable id')).toEqual(['attr', undefined]) + expect(classes(code, '"user@example.com"')).toEqual(['string']) + expect(classes(code, 'value class')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'enum class Color')).toEqual(['keyword', undefined, 'keyword', undefined, 'type']) + expect(classes(code, 'open annotation class')).toEqual(['keyword', undefined, 'keyword', undefined, 'keyword']) + expect(classes(code, 'inline fun ')).toEqual(['keyword', undefined, 'keyword', undefined, 'operator', 'keyword', undefined, 'operator']) + expect(classes(code, 'only(vararg')).toEqual(['function', undefined, 'keyword']) + expect(classes(code, 'data = map.get(0) + value + open')).toEqual([undefined, 'operator', undefined, 'function', undefined, 'number', undefined, 'operator', undefined, 'operator', undefined]) + expect(classes(code, 'result = data')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'as? String ?: z!!')).toEqual(['keyword', undefined, 'type', undefined, 'operator', undefined, 'operator']) + expect(classes(code, 'expect fun platform')).toEqual(['keyword', undefined, 'keyword', undefined, 'function']) + }) + + it('colours trailing-lambda and generic calls but not types before braces', () => { + const code = [ + 'fun identity(x: T): T { return x }', + 'fun run(block: () -> R): R { return block() }', + 'val big = MAX_SIZE {}', + 'setContent { AppTheme { Column { Text("x") } } }', + 'class A : B, Cee {', + 'enum class Color { RED { override fun x() = 1 }, Green }', + 'tasks.withType { useJUnitPlatform() }', + 'val dto = call.receive() + listOf>()', + 'if (a < b && c > (d)) x', + 'val f = fun(x: Int): Int { return x }', + 'try { } catch (e: E) { } finally { }', + 'companion object { init { do { } while (x) } }', + ].join('\n') + expect(classes(code, 'T): T {')).toEqual([undefined]) + expect(classes(code, 'R): R {')).toEqual([undefined]) + expect(classes(code, 'MAX_SIZE {')).toEqual([undefined]) + expect(classes(code, 'setContent { AppTheme { Column {')).toEqual(['function', undefined, 'function', undefined, 'function', undefined]) + expect(classes(code, 'B, Cee {')).toEqual([undefined, 'type', undefined]) + expect(classes(code, 'RED {')).toEqual([undefined]) + expect(classes(code, 'withType {')).toEqual(['function', 'operator', 'type', 'operator', undefined]) + expect(classes(code, 'receive<')).toEqual(['function', 'operator']) + expect(classes(code, 'listOf (d)')).toEqual([undefined, 'operator', undefined, 'operator', undefined, 'operator', undefined]) + expect(classes(code, 'fun(x: Int): Int {')).toEqual(['keyword', undefined, 'type', undefined, 'type', undefined]) + expect(classes(code, 'try { } catch')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, 'finally {')).toEqual(['keyword', undefined]) + expect(classes(code, 'companion object { init { do {')).toEqual(['keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword', undefined]) + }) + + it('classifies numbers, operators, and declarations', () => { + for (const number of ['1_000', '0xFF', '0xFF_EC_DE_5E', '0b1010', '1.5f', '10L', '1e10', '2.5E-3', '42u', '42UL', '3.14']) { + expect(classes(`val n = ${number}`, number), number).toEqual(['number']) + } + const code = 'val v = "1.2.3"; val w = 1.2.3; val ref = String::length; val same = a === b\nfun > List.maxOf(): T = max()\n@Composable fun Greeting(name: String) { Text("Hi") }\nval MAX_SIZE = x1\nclass Box(val size: Int)\nval task = object : Runnable { }' + expect(classes(code, 'w = 1.2.3')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'String::length')).toEqual(['type', 'operator', undefined]) + expect(classes(code, '===')).toEqual(['operator']) + expect(classes(code, 'List.maxOf')).toEqual(['type', 'operator', undefined, 'operator', undefined, 'function']) + expect(classes(code, 'Greeting')).toEqual(['function']) + expect(classes(code, 'Text(')).toEqual(['function', undefined]) + expect(classes(code, 'class Box(val')).toEqual(['keyword', undefined, 'type', undefined, 'keyword']) + expect(classes(code, 'object : Runnable {')).toEqual(['keyword', undefined, 'type', undefined]) + expect(classes(code, 'MAX_SIZE = x1')).toEqual([undefined, 'operator', undefined]) + }) + + it('registers only the requested aliases', () => { + for (const name of ['kotlin', 'kt', 'kts']) { + expect(highlighter.normalizeLanguage(name)).toBe('kotlin') + } + expect(createHighlighter({ languages: [] }).normalizeLanguage('kotlin')).toBe('plaintext') + }) +}) diff --git a/test/lua.test.ts b/test/lua.test.ts new file mode 100644 index 0000000..3930fe8 --- /dev/null +++ b/test/lua.test.ts @@ -0,0 +1,204 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { lua } from '../src/languages/lua' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [lua] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'lua') + +describe('Lua documentation syntax', () => { + it('highlights a realistic Neovim plugin module', () => { + const code = `#!/usr/bin/env lua +-- lua/myplugin/init.lua +local M = {} + +---@class Config +M.defaults = { + enabled = true, + width = 0.5, + icons = { error = "", warn = "" }, +} + +--[[ Setup is idempotent. + Call it as require("myplugin").setup { ... } ]] +function M.setup(opts) + M.config = vim.tbl_deep_extend("force", M.defaults, opts or {}) + vim.opt.number = true + vim.keymap.set("n", "f", function() + require("myplugin.finder").open() + end, { desc = "Find files" }) +end + +local function count(t) + local n = 0 + for _, v in ipairs(t) do + if v ~= nil then n = n + 1 end + end + return n, #t +end + +function M.Picker:render(items, ...) + local lines = [[ +-- not a comment +]] + for i = 1, #items, 2 do + if items[i] == nil then goto continue end + self.buf[#self.buf + 1] = ("%d: %s"):format(i, items[i]) + ::continue:: + end + return lines .. select("#", ...) +end + +M.on_attach = function(client, bufnr) print "attached" end + +return M +` + expect(classes(code, '#!/usr/bin/env lua')).toEqual(['meta']) + expect(classes(code, '-- lua/myplugin/init.lua')).toEqual(['comment']) + expect(classes(code, '---@class Config')).toEqual(['comment']) + expect(classes(code, '--[[ Setup is idempotent.\n Call it as require("myplugin").setup { ... } ]]')).toEqual(['comment']) + expect(classes(code, 'defaults')).toEqual(['property']) + expect(classes(code, 'true')).toEqual(['literal']) + expect(classes(code, '0.5')).toEqual(['number']) + expect(classes(code, 'setup(')).toEqual(['function', undefined]) + expect(classes(code, 'tbl_deep_extend')).toEqual(['function']) + expect(classes(code, 'opt.number')).toEqual(['property', undefined, 'property']) + expect(classes(code, '"f"')).toEqual(['string']) + expect(classes(code, 'local function count')).toEqual(['keyword', undefined, 'keyword', undefined, 'function']) + expect(classes(code, 'v ~= nil')).toEqual([undefined, 'operator', undefined, 'literal']) + expect(classes(code, '#t')).toEqual(['operator', undefined]) + expect(classes(code, 'render')).toEqual(['function']) + expect(classes(code, '[[\n-- not a comment\n]]')).toEqual(['string']) + expect(classes(code, 'goto continue end')).toEqual(['keyword', undefined, 'keyword']) + expect(classes(code, '::continue::')).toEqual(['operator', undefined, 'operator']) + expect(classes(code, 'format')).toEqual(['function']) + expect(classes(code, '"#"')).toEqual(['string']) + expect(classes(code, 'on_attach')).toEqual(['function']) + expect(classes(code, 'print "attached"')).toEqual(['function', undefined, 'string']) + expect(classes(code, 'return M')).toEqual(['keyword', undefined]) + }) + + it('matches long bracket levels for strings and comments', () => { + const level1 = '[=[ a ]] b ]==] c ]=]' + const level2 = '--[==[ "x" ]] ]=] still comment ]==]' + const code = `local s = ${level1}\n${level2}\nlocal t = [==[\n'quote' -- text\n]==]\nlocal after = 1` + expect(classes(code, level1)).toEqual(['string']) + expect(classes(code, level2)).toEqual(['comment']) + expect(classes(code, "[==[\n'quote' -- text\n]==]")).toEqual(['string']) + expect(classes(code, 'local after = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes('x = [[unterminated -- "', '[[unterminated -- "')).toEqual(['string']) + expect(classes('--[[ open\nlocal x = 1', 'local x = 1')).toEqual(['comment']) + }) + + it('opens a block comment only when a long bracket directly follows two dashes', () => { + const code = '---[[ doc line\nlocal a = 1\n--[ not long\nlocal b = 2\n--[=x\nlocal c = 3\n-- [[ spaced\nlocal d = 4 --]]' + expect(classes(code, '---[[ doc line')).toEqual(['comment']) + expect(classes(code, 'local a')).toEqual(['keyword', undefined]) + expect(classes(code, '--[ not long')).toEqual(['comment']) + expect(classes(code, 'local b')).toEqual(['keyword', undefined]) + expect(classes(code, '--[=x')).toEqual(['comment']) + expect(classes(code, '-- [[ spaced')).toEqual(['comment']) + expect(classes(code, 'local d = 4')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes(code, '--]]')).toEqual(['comment']) + }) + + it('never mistakes indexing for a long string', () => { + const code = 'local v = a[b[1]] + t[ [[key]] ] + m[i][j]\nlocal w = 2' + expect(classes(code, 'a[b[1]]')).toEqual([undefined, 'number', undefined]) + expect(classes(code, '[[key]]')).toEqual(['string']) + expect(classes(code, 'm[i][j]')).toEqual([undefined]) + expect(classes(code, 'local w = 2')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes('f[[arg]]', 'f[[arg]]')).toEqual(['function', 'string']) + }) + + it('handles quoted string escapes and keeps delimiters inside', () => { + const escapes = String.raw`"say \"hi\" \\ \x41 \u{48} \065 -- not a comment"` + const single = String.raw`'it\'s "fine" --[[ no ]]'` + const skip = '"first \\z\n second"' + const continued = '"line \\\nnext"' + const code = `local a = ${escapes}\nlocal b = ${single}\nlocal c = ${skip}\nlocal d = ${continued}\nlocal e = "unterminated\nlocal f = 1 -- "quoted" 'text'` + expect(classes(code, escapes)).toEqual(['string']) + expect(classes(code, single)).toEqual(['string']) + expect(classes(code, skip)).toEqual(['string']) + expect(classes(code, continued)).toEqual(['string']) + expect(classes(code, '"unterminated')).toEqual(['string']) + expect(classes(code, 'local f = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes(code, `-- "quoted" 'text'`)).toEqual(['comment']) + expect(classes('x = "a" .. "--b" .. \'c\'', '.. "--b"')).toEqual(['operator', undefined, 'string']) + }) + + it('treats only a leading #! as a shebang', () => { + const code = '#!/usr/bin/env luajit\nprint(#arg, #"str")' + expect(classes(code, '#!/usr/bin/env luajit')).toEqual(['meta']) + expect(classes(code, '#arg')).toEqual(['operator', undefined]) + expect(classes(code, '#"str"')).toEqual(['operator', 'string']) + expect(classes('local n = #!x', '#')).toEqual(['operator']) + }) + + it('classifies keywords, literals, attribs and labels', () => { + const code = 'local fh = assert(io.open(path))\nlocal K < const > = 10\nwhile not done and x or y do break end\nrepeat n = n - 1 until n <= 0\nlocal const, close = nil, false\n::retry:: goto retry' + expect(classes(code, '')).toEqual(['operator', 'keyword', 'operator']) + expect(classes(code, '< const >')).toEqual(['operator', undefined, 'keyword', undefined, 'operator']) + expect(classes(code, 'while not done and x or y do break end')).toEqual(['keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword']) + expect(classes(code, 'repeat')).toEqual(['keyword']) + expect(classes(code, 'until n <= 0')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + expect(classes('local a = x z', 'constant')).toEqual([undefined]) + expect(classes('local a = x z', 'const and')).toEqual([undefined, 'keyword']) + expect(classes(code, 'local const, close = nil, false')).toEqual(['keyword', undefined, 'operator', undefined, 'literal', undefined, 'literal']) + expect(classes(code, '::retry:: goto retry')).toEqual(['operator', undefined, 'operator', undefined, 'keyword', undefined]) + expect(classes('return self.items', 'self')).toEqual([undefined]) + }) + + it('distinguishes declarations, calls, paren-less calls and member access', () => { + const code = 'function obj:method(a) end\nlocal handler = function() end\nM.cb = function() end\nrequire "mod"\nrequire\'mod\'\nsetup { a = 1 }\nobj:send("x")\nlocal y = vim.fn.expand\nlocal z = x == f\nif ok then\n {}\nend' + expect(classes(code, 'method')).toEqual(['function']) + expect(classes(code, 'handler')).toEqual(['function']) + expect(classes(code, 'cb')).toEqual(['function']) + expect(classes(code, 'require "mod"')).toEqual(['function', undefined, 'string']) + expect(classes(code, "require'mod'")).toEqual(['function', 'string']) + expect(classes(code, 'setup {')).toEqual(['function', undefined]) + expect(classes(code, 'send')).toEqual(['function']) + expect(classes(code, 'fn.expand')).toEqual(['property', undefined, 'property']) + expect(classes(code, 'x == f')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'ok then')).toEqual([undefined, 'keyword']) + expect(classes('for _, v in ipairs { 1 } do end', 'ipairs')).toEqual(['function']) + expect(classes('local w = y\n{ 1 }', 'y')).toEqual([undefined]) + }) + + it('recognizes Lua and LuaJIT numbers without swallowing concatenation', () => { + for (const number of ['42', '3.14', '.5', '5.', '1e10', '2.5E-3', '0xFF', '0x1p4', '0xA.8p0', '0x.1P-2', '42LL', '42ull', '0x2AULL', '1i', '12.5i']) { + expect(classes(`local n = ${number}`, number), number).toEqual(['number']) + } + expect(classes('s = 1 .. 2', '1 .. 2')).toEqual(['number', undefined, 'operator', undefined, 'number']) + expect(classes('s = name..1', 'name..1')).toEqual([undefined, 'operator', 'number']) + expect(classes('s = a..b', 'a..b')).toEqual([undefined, 'operator', undefined]) + expect(classes('f(...)', '...')).toEqual(['operator']) + expect(classes('v = x1 + version 1.2.3', 'x1')).toEqual([undefined]) + expect(classes('v = 1.2.3', '1.2.3')).toEqual([undefined]) + }) + + it('classifies operators and leaves punctuation plain', () => { + const code = 'r = a // b % c ^ d << 1 >> 2 & e | f ~ g ~= h >= i <= j == k' + for (const op of ['//', '%', '^', '<<', '>>', '&', '|', '~ g', '~=', '>=', '<=', '==']) { + expect(classes(code, op)[0], op).toBe('operator') + } + expect(classes('t = { a, b; c }', '{ a, b; c }')).toEqual([undefined]) + }) + + it('tolerates Luau type annotations without classifying them', () => { + const code = 'local function add(a: number, b: number): number\n return a + b\nend\ntype Point = { x: number }' + expect(classes(code, 'add')).toEqual(['function']) + expect(classes(code, 'a: number,')).toEqual([undefined]) + expect(classes(code, 'return a + b')).toEqual(['keyword', undefined, 'operator', undefined]) + expect(classes(code, 'type Point')).toEqual([undefined]) + const interp = "`{player.Name} has {data[\"coins\"]} -- it's {n}`" + expect(classes(`print(${interp})\nlocal z = 1`, interp)).toEqual(['string']) + expect(classes(`print(${interp})\nlocal z = 1`, 'local z = 1')).toEqual(['keyword', undefined, 'operator', undefined, 'number']) + }) + + it('registers no aliases', () => { + expect(highlighter.normalizeLanguage('lua')).toBe('lua') + expect(highlighter.normalizeLanguage('luau')).toBe('plaintext') + expect(createHighlighter({ languages: [] }).normalizeLanguage('lua')).toBe('plaintext') + }) +}) diff --git a/test/perl.test.ts b/test/perl.test.ts new file mode 100644 index 0000000..05b8d1a --- /dev/null +++ b/test/perl.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { perl } from '../src/languages/perl' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [perl] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'perl') + +describe('Perl documentation syntax', () => { + it('highlights a realistic module and script', () => { + const code = `#!/usr/bin/perl +use strict; +use warnings; +use 5.010; +use List::Util qw(sum max); + +package My::Counter; + +=head1 NAME + +My::Counter - count "things" # not code + +=cut + +sub new { + my ($class, %args) = @_; + my $self = { count => $args{start} // 0, name => 'it' }; + return bless $self, $class; +} + +sub increment($self, $by = 1) { + $self->{count} += $by; + return $self->{count}; +} + +package main; + +my $counter = My::Counter->new(start => 0x1F); +my @words = qw(alpha beta#gamma delta); +my $total = 0; +for my $i (1..10) { $total += $i ** 2 } +my $avg = $total / scalar(@words) / 2; +my $text = "Total: $total, items: @{[ scalar @words ]} \\" done"; +(my $copy = $text) =~ s/Total/Sum/g; +my @parts = split /,\\s*/, $line; +print STDERR "last index: $#words, pid $$\\n" unless $ENV{QUIET}; +print <<~EOT; + Hello $name, "quoted" # not a comment + EOT +open(my $fh, '<', $file) or die "Can't open $file: $!"; +while (my $line = <$fh>) { $counter->increment; } +__END__ +Data with "quotes" and s/x/y/ +` + expect(classes(code, '#!/usr/bin/perl')).toEqual(['meta']) + expect(classes(code, 'List::Util')).toEqual(['type']) + expect(classes(code, 'qw(sum max)')).toEqual(['string']) + expect(classes(code, '5.010')).toEqual(['number']) + expect(classes(code, '=head1 NAME\n\nMy::Counter - count "things" # not code\n\n=cut')).toEqual(['comment']) + expect(classes(code, 'new')).toEqual(['function']) + expect(classes(code, '%args')).toEqual(['variable']) + expect(classes(code, '@_')).toEqual(['variable']) + expect(classes(code, 'count =>')).toEqual(['property', undefined, 'operator']) + expect(classes(code, 'start} //')).toEqual(['property', undefined, 'operator']) + expect(classes(code, 'bless')).toEqual(['function']) + expect(classes(code, 'increment(')).toEqual(['function', undefined]) + expect(classes(code, 'My::Counter->new')).toEqual(['type', 'operator', 'function']) + expect(classes(code, '0x1F')).toEqual(['number']) + expect(classes(code, 'qw(alpha beta#gamma delta)')).toEqual(['string']) + expect(classes(code, '1..10')).toEqual(['number', 'operator', 'number']) + expect(classes(code, '$total / scalar')).toEqual(['variable', undefined, 'operator', undefined, 'function']) + expect(classes(code, '"Total: $total, items: @{[ scalar @words ]} \\" done"')).toEqual(['string']) + expect(classes(code, 's/Total/Sum/g')).toEqual(['string']) + expect(classes(code, '/,\\s*/')).toEqual(['string']) + expect(classes(code, '"last index: $#words, pid $$\\n"')).toEqual(['string']) + expect(classes(code, '$ENV')).toEqual(['variable']) + expect(classes(code, '<<~EOT')).toEqual(['string']) + expect(classes(code, ' Hello $name, "quoted" # not a comment\n EOT')).toEqual(['string']) + expect(classes(code, 'or die')).toEqual(['keyword', undefined, 'function']) + expect(classes(code, '$!";')).toEqual(['string', undefined]) + expect(classes(code, '__END__\nData with "quotes" and s/x/y/\n')).toEqual(['comment']) + }) + + it('classifies sigil variables, special variables, and dereferences', () => { + const code = 'my $n = $#array + $_ + $0 + $1 + $$; my ($e, $err) = ($@, $!); my @a = @{$ref}; my $s = ${name};\n' + + 'my %env = %ENV; my @args = @ARGV; my $pkg = $Foo::Bar::baz; my $v = $obj->{key}[0]{name}; &callback(\\&handler);' + for (const name of ['$#array', '$_', '$0', '$1', '$$', '$@', '$!', '${name}', '%ENV', '@ARGV', '$Foo::Bar::baz', '&callback', '&handler']) { + expect(classes(code, name), name).toEqual(['variable']) + } + expect(classes(code, '@{$ref}')).toEqual(['variable', undefined, 'variable', undefined]) + expect(classes(code, '$obj->{key}[0]{name}')).toEqual(['variable', 'operator', undefined, 'property', undefined, 'number', undefined, 'property', undefined]) + const deref = 'print $aref->$#*, $aref->@*, keys $href->%*, $+{name}, keys %+, $-[0], @-;\nprint "still code";' + for (const name of ['$#*', '@*', '%*', '%+', '$-', '@-']) expect(classes(deref, name), name).toEqual(['variable']) + expect(classes(deref, '$+{name}')).toEqual(['variable', undefined, 'property', undefined]) + expect(classes(deref, '"still code"')).toEqual(['string']) + }) + + it('keeps modulo and bitwise operators apart from hash and code sigils', () => { + const code = 'my $m = $total % 7; my $p = ($x) % $y; my %copy = %h; my $b = $x & $y; if ($a && $b) { keys %$ref }' + expect(classes(code, '$total % 7')).toEqual(['variable', undefined, 'operator', undefined, 'number']) + expect(classes(code, ') % $y')).toEqual([undefined, 'operator', undefined, 'variable']) + expect(classes(code, '%copy = %h')).toEqual(['variable', undefined, 'operator', undefined, 'variable']) + expect(classes(code, '$x & $y')).toEqual(['variable', undefined, 'operator', undefined, 'variable']) + expect(classes(code, '$a && $b')).toEqual(['variable', undefined, 'operator', undefined, 'variable']) + expect(classes(code, '%$ref')).toEqual(['variable']) + expect(classes('my %inv = map { $h{$_} => $_ } %h;', '%h;')).toEqual(['variable', undefined]) + }) + + it('scans single, double, and backtick strings as one token', () => { + const code = `my $a = 'it\\'s a \\\\ "#" $x'; my $b = "say \\"hi\\" # @{[ $x + 1 ]} \${y}"; my $c = \`ls -l # not\`;` + expect(classes(code, `'it\\'s a \\\\ "#" $x'`)).toEqual(['string']) + expect(classes(code, '"say \\"hi\\" # @{[ $x + 1 ]} ${y}"')).toEqual(['string']) + expect(classes(code, '`ls -l # not`')).toEqual(['string']) + expect(classes(code, '; my $c')).toEqual([undefined, 'keyword', undefined, 'variable']) + }) + + it('handles quote-like operators with any delimiter and nested brackets', () => { + const code = 'my @l = (q(a (b) c), qq{x {$y} z}, qw/a b/, qr/^\\d+$/i, qx(ls));\n' + + '$s =~ m/a.b/gi; $s =~ m{^/path}x; $s =~ m!x!; $s =~ s/a/b/g; $s =~ s{a}{b}g; $s =~ s{(\\w+)} {uc $1}ge;\n' + + '$s =~ s|a|b|; $s =~ tr/a-z/A-Z/; $s =~ y/a/b/; $s =~ s#a#b#; print "done";' + for (const text of ['q(a (b) c)', 'qq{x {$y} z}', 'qw/a b/', 'qr/^\\d+$/i', 'qx(ls)', 'm/a.b/gi', 'm{^/path}x', 'm!x!', 's/a/b/g', 's{a}{b}g', 's{(\\w+)} {uc $1}ge', 's|a|b|', 'tr/a-z/A-Z/', 'y/a/b/', 's#a#b#']) { + expect(classes(code, text), text).toEqual(['string']) + } + expect(classes(code, 'print "done"')).toEqual(['function', undefined, 'string']) + }) + + it('does not open quote-like operators on keys, methods, names, or variables', () => { + const code = 'print $h{s}, $h{q}, $h{ y }; $obj->s(1); $obj->q;\nsub m { 1 }\nmy %o = (s => 1, y => 2, tr => 3);\nprint $q; my $x = $y / 2; my $z = $q / 4; # end' + expect(classes(code, '$h{s}')).toEqual(['variable', undefined, 'property', undefined]) + expect(classes(code, '$h{q}')).toEqual(['variable', undefined, 'property', undefined]) + expect(classes(code, '$obj->s(1)')).toEqual(['variable', 'operator', 'function', undefined, 'number', undefined]) + expect(classes(code, '$obj->q;')).toEqual(['variable', 'operator', 'function', undefined]) + expect(classes(code, 'sub m {')).toEqual(['keyword', undefined, 'function', undefined]) + expect(classes(code, 's => 1, y => 2, tr => 3')).toEqual(['property', undefined, 'operator', undefined, 'number', undefined, 'property', undefined, 'operator', undefined, 'number', undefined, 'property', undefined, 'operator', undefined, 'number']) + expect(classes(code, '$y / 2')).toEqual(['variable', undefined, 'operator', undefined, 'number']) + expect(classes(code, '$q / 4')).toEqual(['variable', undefined, 'operator', undefined, 'number']) + expect(classes(code, '# end')).toEqual(['comment']) + const more = 'my %o = (m => 1, qw => 2); $x->{s}{y} = 1; my @a = (1) x 3;\nsub q { 1 } if (-s($file)) { my $avg = sum(@n) / @n; }\nmy $ok = 1;' + expect(classes(more, 'm => 1, qw =>')).toEqual(['property', undefined, 'operator', undefined, 'number', undefined, 'property', undefined, 'operator']) + expect(classes(more, '$x->{s}{y}')).toEqual(['variable', 'operator', undefined, 'property', undefined, 'property', undefined]) + expect(classes(more, ') x 3')).toEqual([undefined, 'keyword', undefined, 'number']) + expect(classes(more, 'sub q {')).toEqual(['keyword', undefined, 'function', undefined]) + expect(classes(more, '-s($file)')).toEqual(['operator', 'function', undefined, 'variable', undefined]) + expect(classes(more, ') / @n')).toEqual([undefined, 'operator', undefined, 'variable']) + expect(classes(more, '$ok')).toEqual(['variable']) + }) + + it('tells bare regexes from division and defined-or', () => { + const code = 'if ($s =~ /a#b/ && $t !~ /c/) { return /x/ }\nmy @f = split //, $w; my @g = grep { !/^#/ } @l;\n' + + 'my $r = $a / $b / $c; my $u = ($x + 1) / 2; my $v = $h{k} / 3; my $w = $x // 5; $x //= 1; next unless /^\\s*$/;' + expect(classes(code, '/a#b/')).toEqual(['string']) + expect(classes(code, '/c/')).toEqual(['string']) + expect(classes(code, '/x/')).toEqual(['string']) + expect(classes(code, '//, $w')).toEqual(['string', undefined, 'variable']) + expect(classes(code, '/^#/')).toEqual(['string']) + expect(classes(code, '$a / $b / $c')).toEqual(['variable', undefined, 'operator', undefined, 'variable', undefined, 'operator', undefined, 'variable']) + expect(classes(code, ') / 2')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, '} / 3')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, '$x // 5')).toEqual(['variable', undefined, 'operator', undefined, 'number']) + expect(classes(code, '//= 1')).toEqual(['operator', undefined, 'number']) + expect(classes(code, '/^\\s*$/')).toEqual(['string']) + const paths = '$p =~ s#/#::#g; $u =~ m#^/api#; my $m = time / 60; local $" = \', \'; my $s = "a$" . $\' . $h{\'k\'};\nprint "ok";' + for (const text of ['s#/#::#g', 'm#^/api#', '\', \'', '"a$"', "'k'", '"ok"']) expect(classes(paths, text), text).toEqual(['string']) + expect(classes(paths, 'time / 60')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(paths, '$" =')).toEqual(['variable', undefined, 'operator']) + expect(classes(paths, "$' .")).toEqual(['variable', undefined, 'operator']) + }) + + it('never lets an unterminated slash regex span lines', () => { + const code = 'my $half = time / 2;\nmy @x = (1, 2);\nprint "ok";' + expect(classes(code, '@x')).toEqual(['variable']) + expect(classes(code, '"ok"')).toEqual(['string']) + const unclosed = 'my @p = split /,\nmy $n = 1;' + expect(classes(unclosed, '$n')).toEqual(['variable']) + }) + + it('scans heredocs and keeps the opener line as code', () => { + const code = 'print < 1')).toEqual(['operator', undefined, 'number']) + // `<<~` terminators match horizontal indentation only (a `\s*` there went quadratic over blank lines). + const blank = 'print <<~SQL;\n SELECT 1\n\n\t SQL\nmy $after = 1;\n' + 'print <<~X;\n' + '\n'.repeat(30000) + expect(classes(blank, ' SELECT 1\n\n\t SQL')).toEqual(['string']) + expect(classes(blank, '$after')).toEqual(['variable']) + const started = performance.now() + highlighter.tokenize(blank, { lang: 'perl' }) + expect(performance.now() - started).toBeLessThan(1_000) + }) + + it('isolates comments, POD, and data sections', () => { + const code = 'my $last = $#items; # comment with "quote" and s/x/\nmy @w = qw( a#b c );\n=pod\n\nmy $x = "hidden";\n\n=cut\nmy $y = 1;\n=head2 Unterminated\nstill pod' + expect(classes(code, '$#items')).toEqual(['variable']) + expect(classes(code, '# comment with "quote" and s/x/')).toEqual(['comment']) + expect(classes(code, 'qw( a#b c )')).toEqual(['string']) + expect(classes(code, '=pod\n\nmy $x = "hidden";\n\n=cut')).toEqual(['comment']) + expect(classes(code, '$y')).toEqual(['variable']) + expect(classes(code, '=head2 Unterminated\nstill pod')).toEqual(['comment']) + expect(classes('my $x = 1;\n__DATA__\nraw "text"', '__DATA__\nraw "text"')).toEqual(['comment']) + const inline = 'my %k = (__END__ => 1);\nmy $y = "__DATA__";\nprint "code";' + expect(classes(inline, '__END__')).toEqual(['property']) + expect(classes(inline, '"code"')).toEqual(['string']) + }) + + it('highlights keywords, literals, declarations, and numbers', () => { + const code = 'use v5.36.0; use Data::Dumper; local $x = undef; print __PACKAGE__, __LINE__;\n' + + 'BEGIN { our $n = 1_000_000 + 0xFF + 0b1010 + 017 + 0o17 + 1.5e3 + .5 } if ($a eq $b or not $c) { last }\n' + + 'my $ok = $a <=> $b; $s .= "x" x 3; $x ||= 1; my @r = (1...3); my $ver = "1.2.3"; Foo::Bar->new; my $o = __PACKAGE__->new;' + expect(classes(code, 'v5.36.0')).toEqual(['number']) + expect(classes(code, 'Data::Dumper')).toEqual(['type']) + expect(classes(code, 'local $x = undef')).toEqual(['keyword', undefined, 'variable', undefined, 'operator', undefined, 'literal']) + expect(classes(code, '__PACKAGE__, __LINE__')).toEqual(['literal', undefined, 'literal']) + expect(classes(code, 'BEGIN')).toEqual(['keyword']) + for (const number of ['1_000_000', '0xFF', '0b1010', '017', '0o17', '1.5e3', '.5']) { + expect(classes(code, number), number).toEqual(['number']) + } + expect(classes(code, '$a eq $b or not $c')).toEqual(['variable', undefined, 'keyword', undefined, 'variable', undefined, 'keyword', undefined, 'keyword', undefined, 'variable']) + expect(classes(code, '<=>')).toEqual(['operator']) + expect(classes(code, '.= "x" x 3')).toEqual(['operator', undefined, 'string', undefined, 'keyword', undefined, 'number']) + expect(classes(code, '||=')).toEqual(['operator']) + expect(classes(code, '1...3')).toEqual(['number', 'operator', 'number']) + expect(classes(code, 'Foo::Bar->new')).toEqual(['type', 'operator', 'function']) + expect(classes(code, '__PACKAGE__->new')).toEqual(['literal', 'operator', 'function']) + }) + + it('registers only the requested aliases', () => { + expect(highlighter.normalizeLanguage('perl')).toBe('perl') + expect(highlighter.normalizeLanguage('pl')).toBe('perl') + expect(createHighlighter({ languages: [] }).normalizeLanguage('perl')).toBe('plaintext') + }) +}) diff --git a/test/real-doc-fixtures.test.ts b/test/real-doc-fixtures.test.ts index c37c8ba..9689f98 100644 --- a/test/real-doc-fixtures.test.ts +++ b/test/real-doc-fixtures.test.ts @@ -11,7 +11,7 @@ type DocFixture = { } const fixtures = fixtureData.fixtures as Array -const languagesWithoutRealDocFixtures = new Set(['go', 'cpp', 'cmake', 'php']) +const languagesWithoutRealDocFixtures = new Set(['go', 'cpp', 'cmake', 'php', 'csharp', 'dart', 'java', 'kotlin', 'lua', 'perl', 'ruby', 'rust']) describe('real TanStack docs fixtures', () => { it('covers normalized language targets available in TanStack docs', () => { diff --git a/test/ruby.test.ts b/test/ruby.test.ts new file mode 100644 index 0000000..b53af24 --- /dev/null +++ b/test/ruby.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { ruby } from '../src/languages/ruby' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [ruby] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'ruby') + +describe('Ruby documentation syntax', () => { + it('highlights a realistic library example', () => { + const code = `#!/usr/bin/env ruby +# frozen_string_literal: true +require 'json' + +module Shop + class Cart < Base + include Enumerable + attr_reader :items, :owner + MAX_ITEMS = 1_000 + @@count = 0 + + def initialize(owner:, items: []) + @owner = owner + @items = items + @@count += 1 + end + + def self.build(name) = new(owner: name) + + def total + @items.sum { |item| item[:price] * item.fetch(:qty, 1) } + end + + def ==(other) + other.is_a?(Cart) && items == other.items + end + + def to_s + "#{owner}: #{items.map { |i| "<#{i[:name]}>" }.join(", ")}" + end + + def to_sql + <<~SQL.strip + SELECT * FROM carts # not a comment + WHERE owner = '#{owner}' + SQL + end + + def valid? + raise ArgumentError, "too many" if items.size > MAX_ITEMS + !@owner.nil? && $stdout.tty? + end + end +end +` + expect(classes(code, '#!/usr/bin/env ruby')).toEqual(['meta']) + expect(classes(code, '# frozen_string_literal: true')).toEqual(['comment']) + expect(classes(code, 'require')).toEqual(['keyword']) + expect(classes(code, "'json'")).toEqual(['string']) + expect(classes(code, 'Shop')).toEqual(['type']) + expect(classes(code, 'Cart <')).toEqual(['type', undefined, 'operator']) + expect(classes(code, 'Enumerable')).toEqual(['type']) + expect(classes(code, 'attr_reader')).toEqual(['keyword']) + expect(classes(code, ':items')).toEqual(['string']) + expect(classes(code, 'MAX_ITEMS =')).toEqual([undefined, 'operator']) + expect(classes(code, '1_000')).toEqual(['number']) + expect(classes(code, '@@count')).toEqual(['variable']) + expect(classes(code, 'initialize')).toEqual(['function']) + expect(classes(code, 'owner:,')).toEqual(['property', undefined]) + expect(classes(code, 'self.build')).toEqual(['literal', undefined, 'function']) + expect(classes(code, '|item|')).toEqual([undefined]) + expect(classes(code, '.sum')).toEqual([undefined, 'property']) + expect(classes(code, 'fetch')).toEqual(['function']) + expect(classes(code, 'def ==')).toEqual(['keyword', undefined, 'function']) + expect(classes(code, 'is_a?')).toEqual(['function']) + expect(classes(code, '"#{owner}: #{items.map { |i| "<#{i[:name]}>" }.join(", ")}"')).toEqual(['string']) + expect(classes(code, '<<~SQL.strip')).toEqual(['string', undefined, 'property']) + expect(classes(code, "SELECT * FROM carts # not a comment\n WHERE owner = '#{owner}'\n SQL")).toEqual(['string']) + expect(classes(code, 'valid?')).toEqual(['function']) + expect(classes(code, 'ArgumentError')).toEqual(['type']) + expect(classes(code, '!@owner.nil?')).toEqual(['operator', 'variable', undefined, 'property']) + expect(classes(code, '$stdout')).toEqual(['variable']) + }) + + it('finds the end of interpolated strings with nested quotes and braces', () => { + const nested = '"#{h["k"]} and #{items.map { |i| "<#{i}>" }.join}"' + const code = `out = ${nested} + "#{'}'}" + 'it\\'s #{raw}' # done` + expect(classes(code, nested)).toEqual(['string']) + expect(classes(code, `"#{'}'}"`)).toEqual(['string']) + expect(classes(code, "'it\\'s #{raw}'")).toEqual(['string']) + expect(classes(code, '# done')).toEqual(['comment']) + expect(classes('cmd = `ls #{dir}` # c', '`ls #{dir}`')).toEqual(['string']) + const deep = `${'"#{'.repeat(20)}x${'}"'.repeat(20)} # c` + expect(classes(deep, '# c').at(-1)).toBe('comment') + const regexes = `s = "#{q.sub(/"/, '')} #{/['"]/.match(z)} #{x / 2} #{y /2}" + "#{a.gsub(/\\//, "-")}" # c` + expect(classes(regexes, `"#{q.sub(/"/, '')} #{/['"]/.match(z)} #{x / 2} #{y /2}"`)).toEqual(['string']) + expect(classes(regexes, '"#{a.gsub(/\\//, "-")}"')).toEqual(['string']) + expect(classes(regexes, '# c')).toEqual(['comment']) + }) + + it('scans percent literals with nested delimiters and keeps modulo operators', () => { + const code = 'tags = %w[a b] + %i[c d]\nq = %q(it\'s (nested) "x") + %Q{a {#{b}} c} + %(plain)\nre = %r{/path/(\\d+)}i\ncmd = %x(ls #{dir})\nr = a % b\nx %= 2\ns = "%05d" % n' + for (const literal of ['%w[a b]', '%i[c d]', '%q(it\'s (nested) "x")', '%Q{a {#{b}} c}', '%(plain)', '%r{/path/(\\d+)}i', '%x(ls #{dir})']) { + expect(classes(code, literal), literal).toEqual(['string']) + } + expect(classes(code, 'a % b')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'x %= 2')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, '"%05d" % n')).toEqual(['string', undefined, 'operator', undefined]) + expect(classes('puts %w[x y]', '%w[x y]')).toEqual(['string']) + }) + + it('reads heredoc bodies until their terminator line', () => { + const code = `a = <<~SQL.strip + SELECT 1 # "x +SQL +b = <<-EOS + 'unclosed + EOS +c = < { + const code = 'r = /ab+c/i\nok = x =~ /\\d+/\nputs 1 if /foo/.match?(s)\nparts = s.split(/,\\s*/)\nwords = s.split /\\s+/\nq = a / b\navg = total / count\nx /= 2\nz = (a + b) / (c - d) / 2\nh = h[:k] / 2 # / not regex\nm = path !~ %r{^/#}' + for (const regex of ['/ab+c/i', '/\\d+/', '/foo/', '/,\\s*/', '/\\s+/', '%r{^/#}']) { + expect(classes(code, regex), regex).toEqual(['string']) + } + expect(classes(code, 'a / b')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'total / count')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'x /= 2')).toEqual([undefined, 'operator', undefined, 'number']) + expect(classes(code, ') / (c - d) / 2')).toEqual([undefined, 'operator', undefined, 'operator', undefined, 'operator', undefined, 'number']) + expect(classes(code, '] / 2 # / not regex')).toEqual([undefined, 'operator', undefined, 'number', undefined, 'comment']) + expect(classes(code, '.match?')).toEqual([undefined, 'function']) + expect(classes(code, '=~')).toEqual(['operator']) + expect(classes(code, '!~')).toEqual(['operator']) + expect(classes('x = y /2\nz = 1', '/2\nz = 1')).toEqual(['operator', 'number', undefined, 'operator', undefined, 'number']) + }) + + it('does not open regex, percent or heredoc literals in everyday arithmetic', () => { + const code = `x = y / z; w = 1 / 2 +pct = count / total * 100 + [1, 2].sum / 2 + obj.size / 2 + @total / @count + x.to_f / y +puts(a / b / c) if x / 2 > 1 +odd = total /count * 100 / 2 +mid = width /2 - height/2 +m = value % 10 + i % 2 + 10 % 3 + a % (b + c) + h[k] / 2 +list << Item.new +io << "a" << CONSTANT +mask = 1 << 2 +rows = DB.query(<<~SQL, limit: 10) + SELECT * FROM t -- "q" / 50% < classes(code, text).filter(Boolean) + expect(ops('y / z; w = 1 / 2')).toEqual(['operator', 'operator', 'number', 'operator', 'number']) + expect(ops('count / total * 100 + [1, 2].sum / 2 + obj.size / 2 + @total / @count + x.to_f / y')).not.toContain('string') + expect(ops('a / b / c) if x / 2 > 1')).toEqual(['operator', 'operator', 'keyword', 'operator', 'number', 'operator', 'number']) + expect(ops('total /count * 100 / 2')).toEqual(['operator', 'operator', 'number', 'operator', 'number']) + expect(ops('width /2 - height/2')).toEqual(['operator', 'number', 'operator', 'operator', 'number']) + expect(ops('value % 10 + i % 2 + 10 % 3 + a % (b + c) + h[k] / 2')).not.toContain('string') + expect(classes(code, 'list << Item.new')).toEqual([undefined, 'operator', undefined, 'type', undefined, 'property']) + expect(classes(code, 'io << "a" << CONSTANT')).toEqual([undefined, 'operator', undefined, 'string', undefined, 'operator', undefined]) + expect(classes(code, '(<<~SQL, limit: 10)')).toEqual([undefined, 'string', undefined, 'property', undefined, 'number', undefined]) + expect(classes(code, ' SELECT * FROM t -- "q" / 50% < { + const code = 'opts = { key: :value, "str": 1, :"sym bol" => nil, :old=>2 }\nFoo::Bar::BAZ\nx = cond ? first : second\ny = flag ? :yes : :no\nsend(:empty?)' + expect(classes(code, 'key: :value')).toEqual(['property', undefined, 'string']) + expect(classes(code, '"str": 1')).toEqual(['string', undefined, 'number']) + expect(classes(code, ':"sym bol" => nil')).toEqual(['string', undefined, 'operator', undefined, 'literal']) + expect(classes(code, ':old=>2')).toEqual(['string', 'operator', 'number']) + expect(classes(code, 'Foo::Bar::BAZ')).toEqual(['type', 'operator', 'type', 'operator', undefined]) + expect(classes(code, 'cond ? first : second')).toEqual([undefined, 'operator', undefined]) + expect(classes(code, 'flag ? :yes : :no')).toEqual([undefined, 'operator', undefined, 'string', undefined, 'string']) + expect(classes(code, ':empty?')).toEqual(['string']) + }) + + it('keeps comments, block comments, and hashes inside strings apart', () => { + const code = '=begin\nputs "not code" # still doc\n=end\nx = "a # b" # real \'comment\'\ny = \'#{no}\'\nz = "#$1 #@name"\nv = $\' + $" + $~[0] + $0' + expect(classes(code, '=begin\nputs "not code" # still doc\n=end')).toEqual(['comment']) + expect(classes(code, '"a # b"')).toEqual(['string']) + expect(classes(code, "# real 'comment'")).toEqual(['comment']) + expect(classes(code, "'#{no}'")).toEqual(['string']) + expect(classes(code, '"#$1 #@name"')).toEqual(['string']) + expect(classes(code, '$\' + $" + $~')).toEqual(['variable', undefined, 'operator', undefined, 'variable', undefined, 'operator', undefined, 'variable']) + expect(classes(code, '$0')).toEqual(['variable']) + expect(classes('x = 1 =begin', '=begin')).toEqual(['operator', 'keyword']) + }) + + it('classifies keywords, literals, numbers and operators', () => { + const code = 'def name!; end\ndef self.call?(x) = x\nreturn nil unless defined?(y) && x.class == Foo\nnums = [1_000, 0x1F, 0b1010, 0o17, 017, 1.5e3, 3r, 2i, 1..10, 1...10]\nv = 1.2.3\nf = ->(a, b) { a <=> b ** 2 }\nres&.call || @memo ||= true\na === b; @flag &&= false; obj.then { _1 }\nMAX_SIZE = 1' + expect(classes(code, 'name!')).toEqual(['function']) + expect(classes(code, 'self.call?')).toEqual(['literal', undefined, 'function']) + expect(classes(code, 'return nil unless defined?')).toEqual(['keyword', undefined, 'literal', undefined, 'keyword', undefined, 'keyword']) + expect(classes(code, '.class')).toEqual([undefined, 'property']) + expect(classes(code, '.then')).toEqual([undefined, 'property']) + for (const number of ['1_000', '0x1F', '0b1010', '0o17', '017', '1.5e3', '3r', '2i']) { + expect(classes(code, number), number).toEqual(['number']) + } + expect(classes(code, '1..10')).toEqual(['number', 'operator', 'number']) + expect(classes(code, '1...10')).toEqual(['number', 'operator', 'number']) + expect(classes(code, '1.2.3')).toEqual([undefined]) + for (const operator of ['->', '<=>', '**', '&.', '||', '||=', '===', '&&=']) { + expect(classes(code, operator), operator).toEqual(['operator']) + } + expect(classes(code, 'MAX_SIZE')).toEqual([undefined]) + }) + + it('registers only the requested aliases', () => { + expect(highlighter.normalizeLanguage('ruby')).toBe('ruby') + expect(highlighter.normalizeLanguage('rb')).toBe('ruby') + expect(createHighlighter({ languages: [] }).normalizeLanguage('rb')).toBe('plaintext') + }) +}) diff --git a/test/rust.test.ts b/test/rust.test.ts new file mode 100644 index 0000000..8392047 --- /dev/null +++ b/test/rust.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { rust } from '../src/languages/rust' +import { tokenClasses } from './token-classes' + +const highlighter = createHighlighter({ languages: [rust] }) +const classes = (code: string, text: string) => tokenClasses(highlighter, code, text, 'rust') + +const showcase = `//! A tiny in-memory cache. +#![allow(dead_code)] +use std::collections::HashMap; + +/// Errors returned by [\`Cache::get\`]. Don't unwrap them. +#[derive(Debug, Clone, PartialEq)] +pub enum CacheError { + Missing(String), + Expired { age: u64 }, +} + +#[cfg(feature = "serde")] +pub struct Cache<'a, V: Clone> { + entries: HashMap<&'a str, V>, + hits: usize, +} + +impl<'a, V: Clone> Cache<'a, V> { + pub fn new() -> Self { + Self { entries: HashMap::new(), hits: 0 } + } + + pub fn get(&mut self, key: &'a str) -> Result { + let value = self.entries.get(key).cloned(); + self.hits += 1; + value.ok_or_else(|| CacheError::Missing(format!("no key '{}'", key))) + } +} + +pub async fn warm(cache: &mut Cache<'static, String>) -> Option { + let keys: Vec<_> = (0..10).map(|i| i.to_string()).collect::>(); + for key in keys.iter() { + if key.starts_with('1') { continue; } + println!("warming {key}"); + } + let ratio = 1.5e-3f64 * cache.hits as f64; + Some(cache.hits) +} +` + +describe('Rust documentation syntax', () => { + it('highlights a realistic showcase sample', () => { + expect(classes(showcase, '//! A tiny')).toEqual(['comment']) + expect(classes(showcase, "/// Errors returned by [`Cache::get`]. Don't unwrap them.")).toEqual(['comment']) + expect(classes(showcase, '#![allow')).toEqual(['attr']) + expect(classes(showcase, '#[derive(Debug, Clone')).toEqual(['attr', undefined, 'type', undefined, 'type']) + expect(classes(showcase, '"serde"')).toEqual(['string']) + expect(classes(showcase, 'HashMap;')).toEqual(['type', undefined]) + expect(classes(showcase, 'pub enum CacheError')).toEqual(['keyword', undefined, 'keyword', undefined, 'type']) + expect(classes(showcase, "<'a, V: Clone")).toEqual(['operator', 'meta', undefined, 'type']) + expect(classes(showcase, "&'a str")).toEqual(['operator', 'meta', undefined, 'type']) + expect(classes(showcase, 'fn new() -> Self')).toEqual(['keyword', undefined, 'function', undefined, 'operator', undefined, 'type']) + expect(classes(showcase, '&mut self')).toEqual(['operator', 'keyword', undefined, 'keyword']) + expect(classes(showcase, 'entries.get(')).toEqual(['property', undefined, 'function', undefined]) + expect(classes(showcase, 'format!("no key \'{}\'", key)')).toEqual(['function', undefined, 'string', undefined]) + expect(classes(showcase, "'static")).toEqual(['meta']) + expect(classes(showcase, '0..10')).toEqual(['number', 'operator', 'number']) + expect(classes(showcase, '|i|')).toEqual(['operator', undefined, 'operator']) + expect(classes(showcase, 'collect:: { + const code = "impl<'a> Foo<'a> { fn f<'b, T>(x: &'b str) -> &'static str { 'outer: loop { break 'outer; } } }" + expect(classes(code, "impl<'a> Foo<'a> {")).toEqual(['keyword', 'operator', 'meta', 'operator', undefined, 'type', 'operator', 'meta', 'operator', undefined]) + expect(classes(code, "<'b, T>")).toEqual(['operator', 'meta', undefined, 'operator']) + expect(classes(code, "'outer: loop")).toEqual(['meta', undefined, 'keyword']) + expect(classes(code, "break 'outer;")).toEqual(['keyword', undefined, 'meta', undefined]) + for (const char of ["'a'", "'\\x41'", "'\\n'", "'\\''", "'\\u{1F600}'", "'😀'", "b'x'", "b'\\\\'", "'\"'"]) { + expect(classes(`let c = ${char}; let s: &'a str = x;`, char), char).toEqual(['string']) + } + expect(classes("match c { 'a'..='z' => 1, _ => 0 }", "'a'..='z'")).toEqual(['string', 'operator', 'string']) + }) + + it('matches raw string hash counts and byte/C string prefixes', () => { + for (const string of ['r"C:\\path\\"', 'r#"a "quoted" // x"#', 'r##"a "# b"##', 'b"bytes\\n"', 'br#"raw "b""#', 'c"cstr"', 'cr"x"']) { + expect(classes(`let s = ${string}; let n = 1;`, string), string).toEqual(['string']) + expect(classes(`let s = ${string}; let n = 1;`, 'n = 1'), string).toEqual([undefined, 'operator', undefined, 'number']) + } + const multiline = 'let s = "first \\" /* not\nsecond \\\n third // still";\nlet t = 1;' + expect(classes(multiline, multiline.slice(8, multiline.indexOf(';')))).toEqual(['string']) + expect(classes(multiline, 'let t')).toEqual(['keyword', undefined]) + expect(classes('let s = "it\'s \'a fn"; let c = \'"\';', '"it\'s \'a fn"')).toEqual(['string']) + expect(classes('fn largest(list: &[T]) -> &T {}', 'largest(')).toEqual(['function', 'operator', undefined, 'type', 'operator', undefined]) + expect(classes('let s = r#"unterminated " x', 'r#"unterminated " x')).toEqual(['string']) + const adjacent = 'let a = "\\\\"; let r = r"\\"; let h = br##"x"#y"##; let f = format!("{name:>8.3}"); let z = concat!(r"a",b"b");' + for (const string of ['"\\\\"', 'r"\\"', 'br##"x"#y"##', '"{name:>8.3}"', 'r"a"', 'b"b"']) { + expect(classes(adjacent, string), string).toEqual(['string']) + } + expect(classes(adjacent, '; let r =')).toEqual([undefined, 'keyword', undefined, 'operator']) + }) + + it('keeps chars and lifetimes apart when adjacent on one line', () => { + const code = "let c = 'a'; fn f<'a,'b>(x: &'a mut Foo<'_>) { let s = ' '; }" + expect(classes(code, "'a'; fn f<'a,'b>(")).toEqual(['string', undefined, 'keyword', undefined, 'function', 'operator', 'meta', undefined, 'meta', 'operator', undefined]) + expect(classes(code, "&'a mut Foo<'_>")).toEqual(['operator', 'meta', undefined, 'keyword', undefined, 'type', 'operator', 'meta', 'operator']) + expect(classes(code, "' '")).toEqual(['string']) + }) + + it('nests block comments and keeps quotes inside comments', () => { + const code = '/* outer /* inner "x" */ still \'comment\' */ let a = 1; // don\'t "open"\n/** doc */ fn f() {}' + expect(classes(code, '/* outer /* inner "x" */ still \'comment\' */')).toEqual(['comment']) + expect(classes(code, 'let a = 1;')).toEqual(['keyword', undefined, 'operator', undefined, 'number', undefined]) + expect(classes(code, '// don\'t "open"')).toEqual(['comment']) + expect(classes(code, '/** doc */')).toEqual(['comment']) + expect(classes(code, 'fn f()')).toEqual(['keyword', undefined, 'function', undefined]) + expect(classes('/* /* unclosed */ fn x', '/* /* unclosed */ fn x')).toEqual(['comment']) + expect(classes('let x = "// not /* a comment"; let c = \'/\';', '"// not /* a comment"')).toEqual(['string']) + const edges = '/**/ let a = 1;\n/*/ still comment */ let b = 2;\n/// ```\n/// let x = "y"; // it\'s\n/// ```\nfn g() {}' + expect(classes(edges, '/**/ let')).toEqual(['comment', undefined, 'keyword']) + expect(classes(edges, '/*/ still comment */ let')).toEqual(['comment', undefined, 'keyword']) + expect(classes(edges, '/// let x = "y"; // it\'s')).toEqual(['comment']) + expect(classes(edges, 'fn g')).toEqual(['keyword', undefined, 'function']) + }) + + it('marks a leading shebang as meta but not an inner attribute', () => { + const code = '#!/usr/bin/env -S cargo +nightly -Zscript\n#![allow(unused)]\nfn main() {}' + expect(classes(code, '#!/usr/bin/env -S cargo +nightly -Zscript')).toEqual(['meta']) + expect(classes(code, '#![allow(')).toEqual(['attr', undefined]) + expect(classes('#![no_std]', '#![no_std')).toEqual(['attr']) + }) + + it('classifies macros, attributes, and macro variables', () => { + const code = 'macro_rules! square { ($x:expr) => { $x * $x }; }\nlet v = vec![1, 2];\nassert_eq!(a, b);\n#[cfg(all(test, feature = "x"))]\nif a != b {}' + expect(classes(code, 'macro_rules!')).toEqual(['function']) + expect(classes(code, '$x:expr')).toEqual(['variable', undefined]) + expect(classes(code, 'vec![')).toEqual(['function', undefined]) + expect(classes(code, 'assert_eq!')).toEqual(['function']) + expect(classes(code, '#[cfg(all(')).toEqual(['attr', undefined, 'function', undefined]) + expect(classes(code, '"x"')).toEqual(['string']) + expect(classes(code, 'a != b')).toEqual([undefined, 'operator', undefined]) + }) + + it('supports numeric forms without swallowing ranges or tuple access', () => { + for (const number of ['1_000u64', '0xff_u8', '0o77', '0b1010', '1.5e-3f32', '2.0_f64', '1e10', '42usize', '1.']) { + expect(classes(`let n = ${number};`, number), number).toEqual(['number']) + } + expect(classes('let r = 0..=10;', '0..=10')).toEqual(['number', 'operator', 'number']) + expect(classes('let x = t.0.1 + y.0;', 't.0.1 + y.0')).toEqual([undefined, 'operator', undefined]) + expect(classes('let x = foo1 + 1.max(2);', 'foo1')).toEqual([undefined]) + expect(classes('let x = 1.max(2);', '1.max')).toEqual(['number', undefined, 'function']) + }) + + it('classifies keywords, types, literals, and paths', () => { + const code = 'pub(crate) unsafe trait Shape where Self: Sized { type Item; }\nunion IntOrFloat { i: u32, f: f32 }\nlet union = true;\nlet o: Option = None;\nconst MAX_SIZE: usize = std::mem::size_of::();\nlet c = move || x?;' + expect(classes(code, 'pub(crate) unsafe trait Shape where Self')).toEqual(['keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'keyword', undefined, 'type', undefined, 'keyword', undefined, 'type']) + expect(classes(code, 'type Item')).toEqual(['keyword', undefined, 'type']) + expect(classes(code, 'union IntOrFloat')).toEqual(['keyword', undefined, 'type']) + expect(classes(code, 'let union = true')).toEqual(['keyword', undefined, 'operator', undefined, 'literal']) + expect(classes(code, 'Option = None')).toEqual(['type', 'operator', 'type', 'operator', undefined, 'operator', undefined, 'type']) + expect(classes(code, 'MAX_SIZE: usize')).toEqual([undefined, 'type']) + expect(classes(code, 'std::mem::size_of::()')).toEqual([undefined, 'operator', undefined, 'operator', 'function', 'operator', 'operator', 'type', 'operator', undefined]) + expect(classes('struct S { r#type: u8 }', 'r#type: u8')).toEqual([undefined, 'type']) + expect(classes(code, 'move || x?')).toEqual(['keyword', undefined, 'operator', undefined, 'operator']) + }) + + it('registers only the requested aliases', () => { + expect(highlighter.normalizeLanguage('rust')).toBe('rust') + expect(highlighter.normalizeLanguage('rs')).toBe('rust') + expect(createHighlighter({ languages: [] }).normalizeLanguage('rust')).toBe('plaintext') + }) +}) diff --git a/test/token-classes.ts b/test/token-classes.ts new file mode 100644 index 0000000..b3f21ee --- /dev/null +++ b/test/token-classes.ts @@ -0,0 +1,21 @@ +import { expect } from 'vitest' +import type { Highlighter } from '../src/core' + +// Returns the classes of every token overlapping the first occurrence of `text`. +export function tokenClasses( + highlighter: Highlighter, + code: string, + text: string, + lang: string, +) { + const result = highlighter.tokenize(code, { lang }) + expect(result.tokens.map((token) => token.value).join('')).toBe(code) + const start = code.indexOf(text) + expect(start).toBeGreaterThanOrEqual(0) + let offset = 0 + return result.tokens.flatMap((token) => { + const from = offset + offset += token.value.length + return from < start + text.length && offset > start ? [token.className] : [] + }) +}