diff --git a/.changeset/swift-powershell.md b/.changeset/swift-powershell.md new file mode 100644 index 0000000..d81cb31 --- /dev/null +++ b/.changeset/swift-powershell.md @@ -0,0 +1,5 @@ +--- +'@tanstack/highlight': minor +--- + +Add isolated Swift and PowerShell language definitions, default registration, and idiomatic documentation showcases. Support Swift nested comments and raw delimiters and PowerShell quoting, here-strings, variables, cmdlets, and operators. diff --git a/README.md b/README.md index d75ba3a..a7f3576 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Available themes: Aurora X, Dracula, GitHub Dark, GitHub Light, Gruvbox Dark, Gr ## Languages -`apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `python`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. +`apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `powershell`, `python`, `scheme`, `shell`, `sql`, `svelte`, `swift`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. Each language is available from `@tanstack/highlight/languages/`. The aggregate `@tanstack/highlight/languages` entry can tree-shake, while direct subpaths make isolation explicit. Importing only core helpers from the root entry also removes unused language registrations in a compatible bundler. @@ -215,10 +215,10 @@ Local browser bundles, minified with esbuild and compressed independently. KB us | Registration | Minified | Gzip | Brotli | | --- | ---: | ---: | ---: | | Core, no languages | 3.84 KB | 1.82 KB | 1.66 KB | -| Core + TSX | 10.11 KB | 4.26 KB | 3.90 KB | -| Octane MDX + TypeScript | 13.86 KB | 5.59 KB | 5.17 KB | -| Nine-language docs set | 16.04 KB | 6.20 KB | 5.66 KB | -| All 30 languages | 30.72 KB | 10.77 KB | 9.79 KB | +| Core + TSX | 10.16 KB | 4.29 KB | 3.93 KB | +| Octane MDX + TypeScript | 13.91 KB | 5.62 KB | 5.21 KB | +| Nine-language docs set | 16.09 KB | 6.22 KB | 5.68 KB | +| All 32 languages | 36.17 KB | 12.45 KB | 11.27 KB | The following comparison was measured before the 1.0 property-context correction. Re-run the comparison commands below for current timings. diff --git a/docs/assets/swift-powershell/powershell-dark.png b/docs/assets/swift-powershell/powershell-dark.png new file mode 100644 index 0000000..9101245 Binary files /dev/null and b/docs/assets/swift-powershell/powershell-dark.png differ diff --git a/docs/assets/swift-powershell/powershell-light.png b/docs/assets/swift-powershell/powershell-light.png new file mode 100644 index 0000000..8d803fa Binary files /dev/null and b/docs/assets/swift-powershell/powershell-light.png differ diff --git a/docs/assets/swift-powershell/powershell-mobile.png b/docs/assets/swift-powershell/powershell-mobile.png new file mode 100644 index 0000000..6a238b0 Binary files /dev/null and b/docs/assets/swift-powershell/powershell-mobile.png differ diff --git a/docs/assets/swift-powershell/swift-dark.png b/docs/assets/swift-powershell/swift-dark.png new file mode 100644 index 0000000..0296ce3 Binary files /dev/null and b/docs/assets/swift-powershell/swift-dark.png differ diff --git a/docs/assets/swift-powershell/swift-light.png b/docs/assets/swift-powershell/swift-light.png new file mode 100644 index 0000000..cdae949 Binary files /dev/null and b/docs/assets/swift-powershell/swift-light.png differ diff --git a/docs/assets/swift-powershell/swift-mobile.png b/docs/assets/swift-powershell/swift-mobile.png new file mode 100644 index 0000000..8f64fbf Binary files /dev/null and b/docs/assets/swift-powershell/swift-mobile.png differ diff --git a/docs/config.json b/docs/config.json index 86099b9..5de5476 100644 --- a/docs/config.json +++ b/docs/config.json @@ -34,6 +34,7 @@ { "label": "React Integration", "to": "guides/react" }, { "label": "Octane Integration", "to": "guides/octane" }, { "label": "Custom Languages", "to": "guides/custom-languages" }, + { "label": "Swift and PowerShell", "to": "guides/swift-and-powershell" }, { "label": "Bundle Size and Performance", "to": "guides/performance" } ] }, diff --git a/docs/guides/performance.md b/docs/guides/performance.md index 11df7d7..2dedc6d 100644 --- a/docs/guides/performance.md +++ b/docs/guides/performance.md @@ -8,17 +8,21 @@ CI measures selective browser bundles and highlighting performance on real docum ## Bundle profiles -`pnpm run size` builds seventeen browser profiles with esbuild and measures minified, gzip, and Brotli bytes independently. It also checks that helper, adapter, and selective language imports retain only the requested modules. +`pnpm run size` builds nineteen browser profiles with esbuild and measures minified, gzip, and Brotli bytes independently. It also checks that helper, adapter, and selective language imports retain only the requested modules. | Profile | Registered languages | Current gzip | CI budget | | --- | --- | ---: | ---: | | Core | None | 1.82 KB | 2.0 KB | -| TSX | TSX | 4.26 KB | 4.35 KB | -| Octane | TypeScript plus Octane MDX adapter | 5.59 KB | 5.7 KB | -| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.20 KB | 6.3 KB | -| All | All 30 definitions | 10.77 KB | 10.9 KB | +| TSX | TSX | 4.29 KB | 4.35 KB | +| Octane | TypeScript plus Octane MDX adapter | 5.62 KB | 5.7 KB | +| Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.22 KB | 6.3 KB | +| Swift | Swift | 3.38 KB | 3.55 KB | +| PowerShell | PowerShell | 3.27 KB | 3.55 KB | +| All | All 32 definitions | 12.45 KB | 12.7 KB | -KB uses 1,000 bytes. Core helpers imported from the root tree-shake to the same engine size. The standalone theme helper is 695 gzip bytes. +KB uses 1,000 bytes. Core helpers imported from the root tree-shake to the same engine size. The standalone theme helper is 691 gzip bytes. + +Adding Swift and PowerShell raises the all-language gzip bundle from 10.77 KB to 12.45 KB. Their isolated bundles are 3.38 KB and 3.27 KB gzip including core; existing selective budgets are unchanged. Selective profiles are the primary metric. The all-language profile exists to prevent convenience-entry growth from becoming invisible. @@ -26,7 +30,7 @@ Selective profiles are the primary metric. The all-language profile exists to pr The committed corpus contains 334 real code fences sampled from TanStack documentation, with up to twenty samples per normalized language. -`pnpm run bench` measures tokenization, HTML, Markdown, HAST, line numbers, long decorated blocks, and dedicated C++, CMake, and PHP samples. Each profile reports the median of three samples after two warmup passes, with a 1.2 second CI budget. The main highlighting profile processes at least 10,000 blocks. +`pnpm run bench` measures tokenization, HTML, Markdown, HAST, line numbers, long decorated blocks, and dedicated C++, CMake, PHP, Swift, and PowerShell samples. Each profile reports the median of three samples after two warmup passes, with a 1.2 second CI budget. The main highlighting profile processes at least 10,000 blocks. A local before-and-after review used the same minified bundle settings, fixtures, and benchmark harness on macOS arm64 with Node 24.15.0: @@ -76,4 +80,4 @@ Context-aware fixes are welcome when they solve common docs code. A change shoul The correct response to a crossed budget is to inspect the behavior and architecture. Budgets can move when a measured quality improvement justifies the bytes, but the tradeoff must be explicit. -The 1.0 property-context correction adds roughly 230 gzip bytes to the TSX profile without changing core. Local Node 26 gzip results differ slightly from CI compression: the CI docs profile is 6,217 bytes and all languages is 10,838 bytes. Their budgets are 6,300 and 10,900 bytes respectively, retaining a small explicit margin. +The 1.0 property-context correction adds roughly 230 gzip bytes to the TSX profile without changing core. Current local Node 26.10.0 measurements are 6,220 gzip bytes for the docs profile and 12,449 bytes for all languages. Their CI budgets are 6,300 and 12,700 bytes respectively, retaining a small explicit margin. CI measurements may differ slightly with the compression runtime. diff --git a/docs/guides/swift-and-powershell.md b/docs/guides/swift-and-powershell.md new file mode 100644 index 0000000..bc5b2b7 --- /dev/null +++ b/docs/guides/swift-and-powershell.md @@ -0,0 +1,140 @@ +--- +title: Swift and PowerShell +--- + +# Swift and PowerShell + +Two views of an observatory: Swift keeps concurrent observations behind an actor; PowerShell turns them into a typed pipeline. The examples deliberately mix declarations, metadata, raw text, numeric literals, and control flow so the shipped themes have a rich palette to work with. + +Register only the languages you need: + +```ts +import { createHighlighter } from '@tanstack/highlight/core' +import { swift } from '@tanstack/highlight/languages/swift' +import { powershell } from '@tanstack/highlight/languages/powershell' + +const highlighter = createHighlighter({ languages: [swift, powershell] }) +``` + +Use `swift`, `powershell`, `pwsh`, or `ps1` as fence tags. The default entry registers both definitions. See [Themes](themes) to switch colors without changing markup. + +## Swift: concurrent observations + +Nested comments, actor isolation, `async let`, closure parameters, key paths, an extended regex, and exact raw-string delimiters share one block. The example illustrates highlighting; the API URL is a placeholder. + +```swift +import Foundation + +/* An actor owns the cache. + /* Nested notes stay inside this comment. */ + No locks escape into the view. */ +struct Reading: Sendable, Codable { + let station: String + let celsius: Double + var fahrenheit: Double { celsius * 1.8 + 32 } +} + +actor Observatory { + private var cache: [String: Reading] = [:] + + func reading(for station: String) async throws -> Reading { + if let saved = cache[station] { return saved } + let url = URL(string: "https://example.com/observations/\(station)")! + let (data, _) = try await URLSession.shared.data(from: url) + let reading = try JSONDecoder().decode(Reading.self, from: data) + cache[station] = reading + return reading + } +} + +@MainActor +func forecast() async throws { + let observatory = Observatory() + async let coast = observatory.reading(for: "coast") + async let ridge = observatory.reading(for: "ridge") + let readings = try await [coast, ridge] + let warm = readings.filter { $0.celsius > 0 }.map(\.station) + let scale = 0x1.fp+2 + let mask: UInt8 = 0b1111_0000 + let band = 1..<4 + let pattern = #/^(?[a-z]+):\s+(?-?\d+\.\d+)$/# + let legend = ##"A literal \n; an interpolated value: \##(scale)"## + let report = """ + Stations: \(warm.joined(separator: ", ")) + Calibrated: \(scale), mask: \(mask), band: \(band) + """ + print(report, legend, pattern) +} + +#if DEBUG +let preview = Reading(station: "coast", celsius: 18.5) +#endif +``` + +## PowerShell: a report pipeline + +An advanced function combines attributes, splatting, scoped variables, typed records, here-strings, null coalescing, and case-insensitive word operators. The API URL is a placeholder. + +```powershell +#Requires -Version 7.4 +<# Build typed records, then shape a pipeline for display. + Quotes and $variables in this comment stay quiet. #> +function Get-StationReport { + [CmdletBinding()] + param( + [Parameter(Mandatory)] + [ValidateSet('coast', 'ridge')] + [string[]] $Station + ) + + begin { + $script:Endpoint = $env:STATION_API ?? 'https://example.com' + $headers = @{ Accept = 'application/json' } + $budget = 64MB + } + process { + foreach ($name in $Station) { + $request = @{ + Uri = "$script:Endpoint/observations/$name" + Headers = $headers + ErrorAction = 'Stop' + } + try { + $reading = Invoke-RestMethod @request + [pscustomobject]@{ + Station = $name + Celsius = [double] $reading.celsius + Online = $true + } + } + catch { + Write-Warning -Message "Station $name unavailable: $_" + } + } + } + end { + $banner = @" +Forecast ready +Budget: $budget +"@ + $literal = @' +$HOME is literal; # this is text, not a comment. +'@ + Write-Verbose -Message ($banner + $literal) + } +} + +${report-title} = 'Today''s observatory' +Get-StationReport -Station coast, ridge | + Where-Object { $_.Online -and $_.Celsius -ge 0 } | + Sort-Object -Property Celsius -Descending | + Select-Object -First 5 -Property Station, Celsius +``` + +## Preview and scope + +Run `pnpm run report:compare` in the repository and open `artifacts/shiki-comparison.html`. These two showcases appear first, in GitHub Light and Aurora X, alongside Shiki's grammar-based reference. Their canonical sources are `test/showcases/Observatory.swift` and `test/showcases/Get-StationReport.ps1`. + +Interpolation is kept inside the string token. Bare Swift regex literals, arbitrary custom operators, symbol resolution, and full PowerShell command/argument disambiguation are outside the lightweight tokenizer's scope. + +Lexical rules follow [The Swift Programming Language](https://docs.swift.org/swift-book/documentation/the-swift-programming-language/lexicalstructure/) and PowerShell's [quoting rules](https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules) and [language specification](https://learn.microsoft.com/en-us/powershell/scripting/lang-spec/chapter-02). diff --git a/docs/language-support.md b/docs/language-support.md index 319e952..b6b56ec 100644 --- a/docs/language-support.md +++ b/docs/language-support.md @@ -29,11 +29,13 @@ Every language is an isolated definition imported from `@tanstack/highlight/lang | Nginx | `nginx` | - | Directives, variables, URLs, comments | | PHP | `php` | - | PHP tags, attributes, quoted strings, heredoc/nowdoc, optional HTML delegation | | Plaintext | `plaintext` | `text`, `txt`, `-->` | Escaping only | +| PowerShell | `powershell` | `pwsh`, `ps1` | Here-strings, escaped quotes, variables, cmdlets, type literals, word operators | | Python | `python` | `py` | Triple strings, prefixes, decorators, comments | | Scheme | `scheme` | `scm`, `racket` | Comments, strings, forms, literals | | Shell | `shell` | `bash`, `sh`, `zsh`, `cmd`, `console` | Heredocs, parameter expansion, comment boundaries | | SQL | `sql` | - | Strings, comments, common SQL clauses | | Svelte | `svelte` | - | Markup plus optional script/style and expression delegation | +| Swift | `swift` | - | Nested comments, raw/multiline strings, extended regex delimiters, attributes, concurrency keywords | | TOML | `toml` | - | Strings, comments, tables, properties | | TypeScript | `ts` | `typescript`, `angular-ts` | JavaScript scanner plus TypeScript keywords/types | | TSRX | `tsrx` | `octane` | TypeScript, contextual JSX, Octane component shorthand and template directives | diff --git a/docs/reference/languages.md b/docs/reference/languages.md index b2f5132..5abc801 100644 --- a/docs/reference/languages.md +++ b/docs/reference/languages.md @@ -40,11 +40,13 @@ const highlighter = createHighlighter({ | `nginx` | `@tanstack/highlight/languages/nginx` | None | | `php` | `@tanstack/highlight/languages/php` | None | | `plaintext` | `@tanstack/highlight/languages/plaintext` | `text`, `txt`, `-->` | +| `powershell` | `@tanstack/highlight/languages/powershell` | `pwsh`, `ps1` | | `python` | `@tanstack/highlight/languages/python` | `py` | | `scheme` | `@tanstack/highlight/languages/scheme` | `scm`, `racket` | | `shell` | `@tanstack/highlight/languages/shell` | `bash`, `sh`, `zsh`, `cmd`, `console` | | `sql` | `@tanstack/highlight/languages/sql` | None | | `svelte` | `@tanstack/highlight/languages/svelte` | None | +| `swift` | `@tanstack/highlight/languages/swift` | None | | `toml` | `@tanstack/highlight/languages/toml` | None | | `ts` | `@tanstack/highlight/languages/ts` | `typescript`, `angular-ts` | | `tsrx` | `@tanstack/highlight/languages/tsrx` | `octane` | @@ -52,6 +54,8 @@ const highlighter = createHighlighter({ | `vue` | `@tanstack/highlight/languages/vue` | None | | `yaml` | `@tanstack/highlight/languages/yaml` | `yml` | -`@tanstack/highlight/languages` re-exports `apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `python`, `scheme`, `shell`, `sql`, `svelte`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. The barrel is convenient but individual subpaths make bundle intent explicit. +`@tanstack/highlight/languages` re-exports `apache`, `cmake`, `cpp`, `css`, `diff`, `dockerfile`, `ejs`, `env`, `go`, `html`, `http`, `js`, `json`, `jsx`, `markdown`, `mermaid`, `nginx`, `php`, `plaintext`, `powershell`, `python`, `scheme`, `shell`, `sql`, `svelte`, `swift`, `toml`, `ts`, `tsrx`, `tsx`, `vue`, and `yaml`. The barrel is convenient but individual subpaths make bundle intent explicit. See the [language support matrix](../language-support) for the context-aware behavior and current scope of each registration. + +Swift supports nested comments, raw and multiline strings, extended regex literals (`#/…/#`), attributes, concurrency keywords, numeric bases, and common types. PowerShell supports comments, quoted and here-strings, scoped/braced/splat variables, cmdlet names, parameters, type literals, and word operators. String interpolation stays within the string token in both languages; these lightweight definitions do not parse expressions or resolve symbols. Bare Swift regex literals and full PowerShell command/argument context are outside their scope. diff --git a/docs/test-strategy.md b/docs/test-strategy.md index 06d4e1d..1d2c055 100644 --- a/docs/test-strategy.md +++ b/docs/test-strategy.md @@ -27,7 +27,7 @@ The suite protects the package's actual product boundary: valid code commonly pu | TSX | TSX | 4.35 KB | | Octane | TypeScript plus Octane MDX adapter | 5.7 KB | | Docs | CSS, HTML, JS, JSON, JSX, Markdown, Shell, TS, TSX | 6.3 KB | -| All | All 30 definitions | 10.9 KB | +| All | All 32 definitions | 12.7 KB | The selective profiles are the primary product metric. The all-language profile protects the convenience entry from unbounded growth. Bundle graphs reject unexpected language or theme code. Package tests repeat isolation checks through public exports after building. diff --git a/scripts/bench.mjs b/scripts/bench.mjs index 12d8830..9b8760d 100644 --- a/scripts/bench.mjs +++ b/scripts/bench.mjs @@ -56,6 +56,14 @@ try { outputBytes: htmlBytes, targetBlocks: 10_000, }, + ...Object.fromEntries([ + ['swift', 'Observatory.swift'], ['powershell', 'Get-StationReport.ps1'], + ].map(([rawLang, file]) => [rawLang, { + fixtures: [{ rawLang, code: fs.readFileSync(`test/showcases/${file}`, 'utf8') }], + run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), + observe: (result) => result.html.length, + targetBlocks: 2_000, + }])), cpp: { fixtures: [{ rawLang: 'cpp', code: '#include \nconstexpr auto text = R"tag(// raw text)tag";\nint main() { std::vector values{1, 2, 3}; return values.size(); }' }], run: (fixture) => highlight(fixture.code, { lang: fixture.rawLang }), diff --git a/scripts/generate-visual-compare.mjs b/scripts/generate-visual-compare.mjs index c822c9a..945e57e 100644 --- a/scripts/generate-visual-compare.mjs +++ b/scripts/generate-visual-compare.mjs @@ -9,7 +9,13 @@ import { githubLightTheme } from '../dist/themes/github-light.js' const fixtureFile = 'test/generated/tanstack-doc-fixtures.json' const outFile = process.argv[2] || 'artifacts/shiki-comparison.html' const fixtureData = JSON.parse(fs.readFileSync(fixtureFile, 'utf8')) -const fixtures = selectFixtures(fixtureData.fixtures) +const fixtures = selectFixtures([ + ...[['swift', 'Observatory.swift'], ['powershell', 'Get-StationReport.ps1']].map(([lang, file]) => ({ + lang, rawLang: lang, file: `test/showcases/${file}`, line: 1, + code: fs.readFileSync(`test/showcases/${file}`, 'utf8'), + })), + ...fixtureData.fixtures, +]) const shikiHighlighter = await shiki.createHighlighter({ themes: ['github-light', 'aurora-x'], langs: ['plaintext'], diff --git a/scripts/measure-size.mjs b/scripts/measure-size.mjs index 016b84a..c0bd062 100644 --- a/scripts/measure-size.mjs +++ b/scripts/measure-size.mjs @@ -98,13 +98,18 @@ const profiles = { languages: ['cmake'], limits: { minified: 6_000, gzip: 2_800, brotli: 2_600 }, }, + ...Object.fromEntries(['swift', 'powershell'].map((language) => [language, { + source: `import { createHighlighter } from './src/core.ts'; import { ${language} } from './src/languages/${language}.ts'; globalThis.highlighter = createHighlighter({ languages: [${language}] })`, + languages: [language], + limits: { minified: 7_700, gzip: 3_550, brotli: 3_300 }, + }])), all: { source: ` import { defaultHighlighter } from './src/index.ts' globalThis.highlighter = defaultHighlighter `, languages: 'all', - limits: { minified: 30_800, gzip: 10_900, brotli: 9_800 }, + limits: { minified: 36_400, gzip: 12_700, brotli: 11_600 }, }, reactAdapter: { source: `export * from './src/react.ts'`, diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 0a38272..b03f533 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -95,6 +95,13 @@ const isolatedBundles = [ source: `import { createHighlighter } from '@tanstack/highlight/core'; import { tsx } from '${entry}'; globalThis.highlighter = createHighlighter({ languages: [tsx] })`, languages: ['tsx'], })), + ...['swift', 'powershell'].flatMap((language) => [ + `@tanstack/highlight/languages/${language}`, '@tanstack/highlight/languages', + ].map((entry) => ({ + name: `${entry} ${language}`, + source: `import { createHighlighter } from '@tanstack/highlight/core'; import { ${language} } from '${entry}'; globalThis.highlighter = createHighlighter({ languages: [${language}] })`, + languages: [language], + }))), ...['react', 'markdown', 'remark', 'rehype', 'octane'].map((entry) => ({ name: `${entry} adapter`, source: `export * from '@tanstack/highlight/${entry}'`, diff --git a/src/index.ts b/src/index.ts index 2500ca8..a7273e9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,11 +24,13 @@ import { mermaid } from './languages/mermaid.js' import { nginx } from './languages/nginx.js' import { php } from './languages/php.js' import { plaintext } from './languages/plaintext.js' +import { powershell } from './languages/powershell.js' import { python } from './languages/python.js' import { scheme } from './languages/scheme.js' import { shell } from './languages/shell.js' import { sql } from './languages/sql.js' import { svelte } from './languages/svelte.js' +import { swift } from './languages/swift.js' import { toml } from './languages/toml.js' import { ts } from './languages/ts.js' import { tsrx } from './languages/tsrx.js' @@ -56,11 +58,13 @@ export type HighlightLanguage = | 'nginx' | 'php' | 'plaintext' + | 'powershell' | 'python' | 'scheme' | 'shell' | 'sql' | 'svelte' + | 'swift' | 'toml' | 'ts' | 'tsrx' @@ -128,11 +132,13 @@ export const allLanguages = [ nginx, php, plaintext, + powershell, python, scheme, shell, sql, svelte, + swift, toml, ts, tsrx, diff --git a/src/languages/index.ts b/src/languages/index.ts index 6bf5cb1..a768a5e 100644 --- a/src/languages/index.ts +++ b/src/languages/index.ts @@ -17,11 +17,13 @@ export { mermaid } from './mermaid.js' export { nginx } from './nginx.js' export { php } from './php.js' export { plaintext } from './plaintext.js' +export { powershell } from './powershell.js' export { python } from './python.js' export { scheme } from './scheme.js' export { shell } from './shell.js' export { sql } from './sql.js' export { svelte } from './svelte.js' +export { swift } from './swift.js' export { toml } from './toml.js' export { ts } from './ts.js' export { tsrx } from './tsrx.js' diff --git a/src/languages/powershell.ts b/src/languages/powershell.ts new file mode 100644 index 0000000..67fa97f --- /dev/null +++ b/src/languages/powershell.ts @@ -0,0 +1,93 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { patternTokenizer } from '../internal/patterns.js' + +export const powershell = defineLanguage({ + name: 'powershell', + aliases: ['pwsh', 'ps1'], + tokenize: patternTokenizer([ + { collect: collectPowerShellLexicalRanges }, + { className: 'literal', regex: /\$(?:true|false|null)\b/gi }, + { className: 'variable', regex: /\$\{(?:`[\s\S]|[^}`])*\}|[$@](?:[\p{L}\p{N}_?]+:)?[\p{L}\p{N}_?]+|\$[$^]/gu }, + { className: 'type', regex: /\[(?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*(?:\[\])?\]/g }, + { className: 'keyword', regex: /(?|&]=?/g }, + ]), +}) + +function collectPowerShellLexicalRanges(code: string) { + const ranges: Array = [] + for (let index = 0; index < code.length;) { + const start = index + let end = index + let className: TokenRange['className'] = 'string' + if (code[index] === '`') { index += 2; continue } + if (code.startsWith('${', index)) { + end = index + 2 + while (end < code.length && code[end] !== '}') { + end += code[end] === '`' ? 2 : 1 + } + end = Math.min(end + 1, code.length) + className = 'variable' + } else if (code.startsWith('<#', index)) { + const close = code.indexOf('#>', index + 2) + end = close < 0 ? code.length : close + 2 + className = 'comment' + } else if (code[index] === '#' && (index === 0 || /[\s(){}\[\];'"|&]/.test(code[index - 1]))) { + end = index + 1 + while (end < code.length && !/[\r\n]/.test(code[end])) end++ + className = 'comment' + } else if (code[index] === '@' && /['"]/.test(code[index + 1] || '') && /^[ \t]*(?:\r\n|\r|\n)/.test(code.slice(index + 2))) { + const quote = code[index + 1] + const close = new RegExp(`^${quote}@`, 'gm') + close.lastIndex = index + 2 + const match = close.exec(code) + end = match ? match.index + 2 : code.length + } else if (code[index] === '"' || code[index] === "'") { + end = quotedEnd(code, index) + } + if (end > start) { + ranges.push({ start, end, className }) + index = end + } else index++ + } + return ranges +} + +// Keep nested quoted strings in expandable-string subexpressions protected. +function quotedEnd(code: string, start: number, depth = 0): number { + const quote = code[start] + let index = start + 1 + while (index < code.length) { + if (quote === '"' && code[index] === '`') index += 2 + else if (code[index] === quote) { + index++ + if (code[index] === quote) index++ + else return index + } else if (quote === '"' && code.startsWith('$(', index) && depth < 24) { + let balance = 1 + index += 2 + while (index < code.length && balance) { + if (code.startsWith('<#', index)) { + const close = code.indexOf('#>', index + 2) + index = close < 0 ? code.length : close + 2 + } else if (code[index] === '#' && /[\s(){}\[\];'"|&]/.test(code[index - 1])) { + while (index < code.length && !/[\r\n]/.test(code[index])) index++ + } else if (code[index] === '`') index += 2 + else if (code[index] === '"' || code[index] === "'") index = quotedEnd(code, index, depth + 1) + else { + if (code[index] === '(') balance++ + if (code[index] === ')') balance-- + index++ + } + } + } else index++ + } + return Math.min(index, code.length) +} diff --git a/src/languages/swift.ts b/src/languages/swift.ts new file mode 100644 index 0000000..495614e --- /dev/null +++ b/src/languages/swift.ts @@ -0,0 +1,97 @@ +import { defineLanguage, type TokenRange } from '../core.js' +import { patternTokenizer } from '../internal/patterns.js' + +export const swift = defineLanguage({ + name: 'swift', + tokenize: patternTokenizer([ + { collect: collectSwiftLexicalRanges }, + { className: 'meta', regex: /#(?:if|elseif|else|endif|available|unavailable|selector|keyPath|sourceLocation|warning|error|fileID|filePath|file|line|column|function)\b/g }, + { className: 'attr', regex: /@[A-Za-z_]\w*/g }, + { className: 'variable', regex: /\$(?:\d+|[A-Za-z_]\w*)/g }, + { className: 'literal', regex: /\b(?:true|false|nil)\b/g }, + { className: 'keyword', regex: /\b(?:actor|any|as|associatedtype|async|await|borrowing|break|case|catch|class|consuming|continue|convenience|copy|default|defer|deinit|didSet|distributed|do|dynamic|each|else|enum|extension|fallthrough|fileprivate|final|for|func|get|guard|if|import|indirect|infix|init|inout|internal|in|is|isolated|lazy|let|mutating|nonisolated|nonmutating|open|operator|optional|override|package|postfix|precedencegroup|prefix|private|protocol|public|repeat|required|rethrows|return|self|set|some|static|struct|subscript|super|switch|throw|throws|try|typealias|unowned|var|weak|where|while|willSet)\b/g }, + { className: 'type', regex: /\b(?:actor|class|enum|protocol|struct|typealias)\s+([\p{L}_][\p{L}\p{N}_]*)/gu, group: 1 }, + { className: 'type', regex: /\b(?:Any|AnyObject|Array|Bool|Character|Dictionary|Double|Float|Int(?:8|16|32|64)?|Never|Optional|Result|Self|Set|String|UInt(?:8|16|32|64)?|Void)\b/g }, + { className: 'function', regex: /[\p{L}_][\p{L}\p{N}_]*(?=\s*\()/gu }, + { className: 'number', regex: /(?:(?|[+\-*/%=!<>?&|^~]+/g }, + ]), +}) + +function collectSwiftLexicalRanges(code: string) { + const ranges: Array = [] + for (let index = 0; index < code.length;) { + const start = index + let end = index + let className: TokenRange['className'] = 'string' + if (code.startsWith('//', index)) { + end = index + 2 + while (end < code.length && !/[\r\n]/.test(code[end])) end++ + className = 'comment' + } else if (code.startsWith('/*', index)) { + end = commentEnd(code, index) + className = 'comment' + } else if (code[index] === '`') { + const close = code.indexOf('`', index + 1) + end = close < 0 ? code.length : close + 1 + // Shield raw identifiers from keyword and literal patterns. + className = 'variable' + } else { + end = stringEnd(code, index) + } + // A negative end skips a rejected raw delimiter without emitting a token. + if (end < 0) { index = -end; continue } + if (end > start) { + ranges.push({ start, end, className }) + index = end + } else index++ + } + return ranges +} + +function commentEnd(code: string, start: number) { + let depth = 1 + let index = start + 2 + while (index < code.length && depth) { + if (code.startsWith('/*', index)) { depth++; index += 2 } + else if (code.startsWith('*/', index)) { depth--; index += 2 } + else index++ + } + return index +} + +function stringEnd(code: string, start: number, depth = 0): number { + let index = start + while (code[index] === '#') index++ + const hashes = code.slice(start, index) + const regex = Boolean(hashes) && code[index] === '/' + if (code[index] !== '"' && !regex) return hashes ? -index : start + const quote = regex ? '/' : code.startsWith('"""', index) ? '"""' : '"' + index += quote.length + const close = quote + hashes + const escape = regex ? '\\' : '\\' + hashes + while (index < code.length) { + if (code.startsWith(close, index)) return index + close.length + if (code.startsWith(escape, index)) { + index += escape.length + if (!regex && code[index] === '(' && depth < 24) { + let balance = 1 + index++ + while (index < code.length && balance) { + const nested = code.startsWith('/*', index) ? commentEnd(code, index) : stringEnd(code, index, depth + 1) + if (nested < 0) index = -nested + else if (nested > index) index = nested + else if (code.startsWith('//', index)) { + while (index < code.length && !/[\r\n]/.test(code[index])) index++ + } else { + if (code[index] === '(') balance++ + if (code[index] === ')') balance-- + index++ + } + } + } else index++ + } else index++ + } + return code.length +} diff --git a/test/fixtures.ts b/test/fixtures.ts index f4ef90a..a4948bc 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs' import type { HighlightLanguage } from '../src/index' export type LanguageFixture = { @@ -8,6 +9,16 @@ export type LanguageFixture = { } export const languageFixtures: Array = [ + { + lang: 'swift', normalized: 'swift', + code: readFileSync(new URL('./showcases/Observatory.swift', import.meta.url), 'utf8'), + expectedClasses: ['th-keyword', 'th-type', 'th-attr', 'th-variable', 'th-string', 'th-comment', 'th-meta', 'th-number', 'th-function', 'th-operator', 'th-property'], + }, + { + lang: 'pwsh', normalized: 'powershell', + code: readFileSync(new URL('./showcases/Get-StationReport.ps1', import.meta.url), 'utf8'), + expectedClasses: ['th-keyword', 'th-type', 'th-variable', 'th-string', 'th-comment', 'th-literal', 'th-command', 'th-number', 'th-function', 'th-operator', 'th-property'], + }, { lang: 'php', normalized: 'php', diff --git a/test/real-doc-fixtures.test.ts b/test/real-doc-fixtures.test.ts index c37c8ba..a4b40a2 100644 --- a/test/real-doc-fixtures.test.ts +++ b/test/real-doc-fixtures.test.ts @@ -11,7 +11,7 @@ type DocFixture = { } const fixtures = fixtureData.fixtures as Array -const languagesWithoutRealDocFixtures = new Set(['go', 'cpp', 'cmake', 'php']) +const languagesWithoutRealDocFixtures = new Set(['go', 'cpp', 'cmake', 'php', 'swift', 'powershell']) describe('real TanStack docs fixtures', () => { it('covers normalized language targets available in TanStack docs', () => { diff --git a/test/showcases/Get-StationReport.ps1 b/test/showcases/Get-StationReport.ps1 new file mode 100644 index 0000000..415d0ab --- /dev/null +++ b/test/showcases/Get-StationReport.ps1 @@ -0,0 +1,53 @@ +#Requires -Version 7.4 +<# Build typed records, then shape a pipeline for display. + Quotes and $variables in this comment stay quiet. #> +function Get-StationReport { + [CmdletBinding()] + param( + [Parameter(Mandatory)] + [ValidateSet('coast', 'ridge')] + [string[]] $Station + ) + + begin { + $script:Endpoint = $env:STATION_API ?? 'https://example.com' + $headers = @{ Accept = 'application/json' } + $budget = 64MB + } + process { + foreach ($name in $Station) { + $request = @{ + Uri = "$script:Endpoint/observations/$name" + Headers = $headers + ErrorAction = 'Stop' + } + try { + $reading = Invoke-RestMethod @request + [pscustomobject]@{ + Station = $name + Celsius = [double] $reading.celsius + Online = $true + } + } + catch { + Write-Warning -Message "Station $name unavailable: $_" + } + } + } + end { + $banner = @" +Forecast ready +Budget: $budget +"@ + $literal = @' +$HOME is literal; # this is text, not a comment. +'@ + Write-Verbose -Message ($banner + $literal) + } +} + +${report-title} = 'Today''s observatory' +Get-StationReport -Station coast, ridge | + Where-Object { $_.Online -and $_.Celsius -ge 0 } | + Sort-Object -Property Celsius -Descending | + Select-Object -First 5 -Property Station, Celsius diff --git a/test/showcases/Observatory.swift b/test/showcases/Observatory.swift new file mode 100644 index 0000000..42df925 --- /dev/null +++ b/test/showcases/Observatory.swift @@ -0,0 +1,46 @@ +import Foundation + +/* An actor owns the cache. + /* Nested notes stay inside this comment. */ + No locks escape into the view. */ +struct Reading: Sendable, Codable { + let station: String + let celsius: Double + var fahrenheit: Double { celsius * 1.8 + 32 } +} + +actor Observatory { + private var cache: [String: Reading] = [:] + + func reading(for station: String) async throws -> Reading { + if let saved = cache[station] { return saved } + let url = URL(string: "https://example.com/observations/\(station)")! + let (data, _) = try await URLSession.shared.data(from: url) + let reading = try JSONDecoder().decode(Reading.self, from: data) + cache[station] = reading + return reading + } +} + +@MainActor +func forecast() async throws { + let observatory = Observatory() + async let coast = observatory.reading(for: "coast") + async let ridge = observatory.reading(for: "ridge") + let readings = try await [coast, ridge] + let warm = readings.filter { $0.celsius > 0 }.map(\.station) + let scale = 0x1.fp+2 + let mask: UInt8 = 0b1111_0000 + let band = 1..<4 + let pattern = #/^(?[a-z]+):\s+(?-?\d+\.\d+)$/# + let legend = ##"A literal \n; an interpolated value: \##(scale)"## + let report = """ + Stations: \(warm.joined(separator: ", ")) + Calibrated: \(scale), mask: \(mask), band: \(band) + """ + print(report, legend, pattern) +} + +#if DEBUG +let preview = Reading(station: "coast", celsius: 18.5) +#endif diff --git a/test/swift-powershell.test.ts b/test/swift-powershell.test.ts new file mode 100644 index 0000000..f763eca --- /dev/null +++ b/test/swift-powershell.test.ts @@ -0,0 +1,126 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { createHighlighter } from '../src/core' +import { normalizeLanguage } from '../src/index' +import { swift } from '../src/languages/swift' +import { powershell } from '../src/languages/powershell' +import { markdown } from '../src/languages/markdown' + +const highlighter = createHighlighter({ languages: [swift, powershell, markdown] }) +function classes(code: string, text: string, lang: string) { + const result = highlighter.tokenize(code, { lang }) + expect(result.tokens.map((token) => token.value).join('')).toBe(code) + const start = code.indexOf(text) + expect(start).toBeGreaterThanOrEqual(0) + let offset = 0 + return result.tokens.flatMap((token) => { + const from = offset + offset += token.value.length + return from < start + text.length && offset > start ? [token.className] : [] + }) +} + +describe('Swift documentation syntax', () => { + it('protects nested comments, raw identifiers and exact raw delimiters', () => { + const comment = '/* outer /* inner */ "still" */' + const raw = '##"quote "# // not a comment \\n"##' + const code = `${comment}\nlet value = ${raw}\nlet \`class\` = nil` + expect(classes(code, comment, 'swift')).toEqual(['comment']) + expect(classes(code, raw, 'swift')).toEqual(['string']) + expect(classes(code, '`class`', 'swift')).toEqual(['variable']) + expect(classes(code, 'nil', 'swift')).toEqual(['literal']) + }) + it('keeps multiline strings and nested interpolation expressions intact', () => { + for (const text of ['"""\n// text\n"quote"\n"""', '"Hello \\(names.joined(separator: ", "))!"', '#"Hello \\#(format("x"))"#', '#/a["/]b\\d+/#']) { + expect(classes(`let text = ${text}\nlet ready = true`, text, 'swift')).toEqual(['string']) + } + }) + it('uses ordinary escapes inside extended regex delimiters', () => { + for (const text of [String.raw`#/foo\/#bar/#`, String.raw`#/foo\#/#`]) { + const code = `let pattern = ${text}\nlet ready = true` + expect(classes(code, text, 'swift')).toEqual(['string']) + expect(classes(code, 'true', 'swift')).toEqual(['literal']) + } + }) + it('skips long rejected hash runs outside strings and inside interpolation', () => { + const hashes = '#'.repeat(100_000) + for (const code of [hashes, `${hashes}x\nlet ready = true`, `"Value: \\(${hashes}x)"\nlet ready = true`]) { + const result = highlighter.tokenize(code, { lang: 'swift' }) + expect(result.tokens.map((token) => token.value).join('')).toBe(code) + if (code.includes('true')) expect(classes(code, 'true', 'swift')).toEqual(['literal']) + if (code.startsWith('"')) expect(classes(code, code.split('\n')[0], 'swift')).toEqual(['string']) + else expect(result.tokens.filter((token) => token.className === 'string')).toEqual([]) + } + }, 1000) + it('preserves raw strings and regexes after long hash delimiters', () => { + const hashes = '#'.repeat(256) + for (const text of [`${hashes}"value"${hashes}`, `${hashes}/value/${hashes}`]) { + expect(classes(`let value = ${text}\nlet ready = true`, text, 'swift')).toEqual(['string']) + } + }) + it('recognizes concurrency, attributes, types, numeric bases and ranges', () => { + const code = '@MainActor\nactor Cache { func read() async throws -> Int { try await load() } }\n#if DEBUG\n#endif' + for (const word of ['actor', 'async', 'throws', 'try', 'await']) expect(classes(code, word, 'swift')).toEqual(['keyword']) + expect(classes(code, '@MainActor', 'swift')).toEqual(['attr']) + expect(classes(code, 'Cache', 'swift')).toEqual(['type']) + expect(classes(code, '#if', 'swift')).toEqual(['meta']) + for (const number of ['0b1010_0011', '0o755', '0xFF', '0x1.fp+2', '1_000.5e-2', '42']) expect(classes(`let x = ${number}`, number, 'swift')).toEqual(['number']) + expect(classes('1..<4', '..<', 'swift')).toEqual(['operator']) + expect(classes('1...4', '...', 'swift')).toEqual(['operator']) + expect(classes('1...4', '4', 'swift')).toEqual(['number']) + expect(classes('let ratio = total / count', '/', 'swift')).toEqual(['operator']) + }) +}) + +describe('PowerShell documentation syntax', () => { + it('protects quoted escapes, doubled quotes and comment boundaries', () => { + for (const text of ["'Today''s # forecast'", '"quote `" # text"', '"C:\\cache\\"']) expect(classes(`$x = ${text}`, text, 'ps1')).toEqual(['string']) + expect(classes('Write-Output hello#there # comment', '#there', 'pwsh')).not.toContain('comment') + expect(classes('Write-Output hello#there # comment', '# comment', 'pwsh')).toEqual(['comment']) + expect(classes('Write-Output escaped`#hash', '#hash', 'pwsh')).not.toContain('comment') + expect(classes('<# "text" $true #> $false', '<# "text" $true #>', 'pwsh')).toEqual(['comment']) + }) + it('protects nested expandable-string expressions and braced variable names', () => { + const text = '\"Result: $(Get-Item -Path \"coast\").Name\"' + expect(classes(`$x = ${text}`, text, 'pwsh')).toEqual(['string']) + const commented = '"Value: $(1 <# ) " #> + 2)"' + expect(classes(`$x = ${commented}`, commented, 'pwsh')).toEqual(['string']) + const variable = '${name"#with`}"characters}' + expect(classes(`${variable} = 1`, variable, 'pwsh')).toEqual(['variable']) + }) + it('matches here-string terminators only at the start of a line', () => { + for (const quote of ['"', "'"]) { + const text = `@${quote} \t\r\n# literal $x\r\n ${quote}@ still text\r\n${quote}@` + expect(classes(`$x = ${text}\r\nGet-Item`, text, 'powershell')).toEqual(['string']) + expect(classes(`$x = ${text}\r\nGet-Item`, 'Get-Item', 'powershell')).toEqual(['command']) + } + }) + it('recognizes case-insensitive keywords, variables, parameters and operators', () => { + const code = 'FUNCTION Get-Report { PARAM([string[]] $Name) $script:Count = $ENV:HOME; ${a-b} = @params; IF ($TRUE -AND $? -cnotmatch "x") { Get-Item -LiteralPath $Name } }' + for (const word of ['FUNCTION', 'PARAM', 'IF']) expect(classes(code, word, 'pwsh')).toEqual(['keyword']) + for (const word of ['$script:Count', '$ENV:HOME', '${a-b}', '@params', '$?']) expect(classes(code, word, 'pwsh')).toEqual(['variable']) + for (const command of ['ForEach-Object', 'Test-Match', 'Compare-In']) expect(classes(command, command, 'pwsh')).toEqual(['command']) + for (const word of ['-AND', '-cnotmatch']) expect(classes(code, word, 'pwsh')).toEqual(['operator']) + expect(classes(code, '$TRUE', 'pwsh')).toEqual(['literal']) + expect(classes(code, '-LiteralPath', 'pwsh')).toEqual(['property']) + expect(classes(code, '[string[]]', 'pwsh')).toEqual(['type']) + expect(classes(code, 'Get-Report', 'pwsh')).toEqual(['function']) + expect(classes('1..4', '4', 'pwsh')).toEqual(['number']) + for (const number of ['64MB', '0xFF', '0b1010', '1.2e3', '42UL']) expect(classes(`$x = ${number}`, number, 'pwsh')).toEqual(['number']) + }) +}) + +it('registers default aliases, isolates imports and delegates Markdown fences', () => { + for (const name of ['powershell', 'pwsh', 'ps1']) expect(normalizeLanguage(name)).toBe('powershell') + expect(normalizeLanguage('swift')).toBe('swift') + expect(createHighlighter({ languages: [swift] }).normalizeLanguage('pwsh')).toBe('plaintext') + for (const [lang, source, text] of [['swift', 'actor Cache {}', 'actor'], ['pwsh', 'Get-Item $HOME', 'Get-Item']]) { + expect(classes(`\`\`\`${lang}\n${source}\n\`\`\``, text, 'markdown')).toEqual([lang === 'swift' ? 'keyword' : 'command']) + } +}) + +it('keeps the PowerShell guide identical to its canonical showcase', () => { + const guide = readFileSync(new URL('../docs/guides/swift-and-powershell.md', import.meta.url), 'utf8') + const showcase = readFileSync(new URL('./showcases/Get-StationReport.ps1', import.meta.url), 'utf8') + expect(guide.match(/```powershell\n([\s\S]*?)\n```/)?.[1]).toBe(showcase.trimEnd()) +})