diff --git a/package-lock.json b/package-lock.json index df633f23259..fae4754c26d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9776,6 +9776,15 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/highlight.js": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.12.0.tgz", + "integrity": "sha512-nbfWpyRMcMrPMmDwJB+dhX/eiaPKtc2RB+0QZskqJ3WjRA/FDS0e9hZrx8EC/lbEv8gXy98FcDbNa/dspAaJMg==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/hono": { "version": "4.13.13", "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.13.tgz", @@ -16352,7 +16361,8 @@ "@maka/mcp": "0.1.0", "@maka/runtime": "0.1.0", "@maka/runtime-host": "0.1.0", - "@maka/storage": "0.1.0" + "@maka/storage": "0.1.0", + "highlight.js": "11.12.0" }, "bin": { "maka": "dist/cli.js" diff --git a/packages/cli/README.md b/packages/cli/README.md index a0e3c53675b..453cfc20ed8 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -288,6 +288,18 @@ on your selected branch. It explains Session coordination; it does not install M npx skills add apache/maka --skill maka-cli ``` +## Code blocks in the TUI + +The live conversation and detailed transcript reader highlight tagged Markdown code blocks using +the terminal's existing color settings. `NO_COLOR` and `TERM=dumb` disable highlighting colors. + +Supported languages are Bash (`sh`, `shell`, `zsh`), C, C++ (`c++`), C# (`cs`), CSS, diff, Go, +Java, JavaScript (`js`, `jsx`, `mjs`, `cjs`), JSON, Markdown (`md`), PowerShell (`ps1`), +Python (`py`), Rust (`rs`), SQL, TypeScript (`ts`, `tsx`), XML (`html`, `htm`, `svg`), and +YAML (`yml`). Tags are case-insensitive. Missing or unknown tags, highlighting failures, and +blocks longer than 4,096 UTF-16 code units render as plain code. Language auto-detection is +disabled; highlighting never changes stored message content. + ## Upgrade Update within the nightly line by pinning the exact release (`--target` accepts `latest`, diff --git a/packages/cli/README.zh-CN.md b/packages/cli/README.zh-CN.md index 55bfdbf8564..dccbc0298e9 100644 --- a/packages/cli/README.zh-CN.md +++ b/packages/cli/README.zh-CN.md @@ -262,6 +262,17 @@ skill 本身不安装 Maka CLI： npx skills add apache/maka --skill maka-cli ``` +## TUI 代码块 + +实时对话和详细记录阅读器会按终端现有的颜色设置，高亮带语言标签的 Markdown 代码块。 +设置 `NO_COLOR` 或 `TERM=dumb` 可禁用高亮颜色。 + +支持 Bash（`sh`、`shell`、`zsh`）、C、C++（`c++`）、C#（`cs`）、CSS、diff、Go、Java、 +JavaScript（`js`、`jsx`、`mjs`、`cjs`）、JSON、Markdown（`md`）、PowerShell（`ps1`）、 +Python（`py`）、Rust（`rs`）、SQL、TypeScript（`ts`、`tsx`）、XML（`html`、`htm`、`svg`） +和 YAML（`yml`），标签不区分大小写。标签缺失或未知、高亮失败，以及长度超过 4,096 个 +UTF-16 代码单元的代码块均显示为纯文本。不自动检测语言，高亮不会修改保存的消息内容。 + ## 升级 在 nightly 线内更新时固定精确版本（`--target` 只接受 `latest`、`next` 或精确的 Maka diff --git a/packages/cli/THIRD_PARTY_NOTICES.txt b/packages/cli/THIRD_PARTY_NOTICES.txt index 68eca268d26..90fb3e220a6 100644 --- a/packages/cli/THIRD_PARTY_NOTICES.txt +++ b/packages/cli/THIRD_PARTY_NOTICES.txt @@ -4605,6 +4605,44 @@ SOFTWARE. ================================================================================ +Package: highlight.js@11.12.0 +Declared license: BSD-3-Clause +Selected license: BSD-3-Clause +Repository: git://github.com/highlightjs/highlight.js.git + +--- LICENSE --- +BSD 3-Clause License + +Copyright (c) 2006, Ivan Sagalaev. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + Package: html-escaper@3.0.3 Declared license: MIT Selected license: MIT diff --git a/packages/cli/package.json b/packages/cli/package.json index e1e2613acd7..8fb64a0ddaa 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -26,6 +26,7 @@ "@maka/mcp": "0.1.0", "@maka/runtime": "0.1.0", "@maka/runtime-host": "0.1.0", - "@maka/storage": "0.1.0" + "@maka/storage": "0.1.0", + "highlight.js": "11.12.0" } } diff --git a/packages/cli/src/__tests__/pi-transcript-highlight.test.ts b/packages/cli/src/__tests__/pi-transcript-highlight.test.ts new file mode 100644 index 00000000000..aaeff0d92ad --- /dev/null +++ b/packages/cli/src/__tests__/pi-transcript-highlight.test.ts @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { test } from 'node:test'; +import { markdownTheme } from '../pi-transcript-format.js'; +import { stripAnsi } from '../tui-ansi.js'; +import { highlightMarkdownCode } from '../tui-syntax-highlight.js'; + +const languageExamples = { + bash: 'echo "你好<&>" # comment', + c: 'const char *label = "你好<&>";', + cpp: 'constexpr auto label = "你好<&>";', + csharp: 'public string label = "你好<&>";', + css: '.label { color: red; content: "你好<&>"; }', + diff: '-old text\n+你好<&> new text', + go: 'func main() { fmt.Println("你好<&>") }', + java: 'public class Label { String text = "你好<&>"; }', + javascript: 'const label = "你好<&>";', + json: '{"label": "你好<&>", "enabled": true}', + markdown: '# 你好<&>\n**bold** and `code`', + powershell: 'Write-Host "你好<&>" # comment', + python: 'def label():\n return "你好<&>"', + rust: 'fn main() { let label = "你好<&>"; }', + sql: "SELECT '你好<&>' FROM labels WHERE id = 1;", + typescript: 'const label: string = "你好<&>";', + xml: '', + yaml: 'label: "你好<&>"\nenabled: true', +} as const; + +const aliases = { + sh: 'bash', + shell: 'bash', + zsh: 'bash', + 'c++': 'cpp', + cs: 'csharp', + html: 'xml', + htm: 'xml', + svg: 'xml', + js: 'javascript', + jsx: 'javascript', + mjs: 'javascript', + cjs: 'javascript', + md: 'markdown', + ps1: 'powershell', + py: 'python', + rs: 'rust', + ts: 'typescript', + tsx: 'typescript', + yml: 'yaml', +} as const; + +// Force visible ANSI in these checks even when the test runner pipes stdout. +// The production theme's environment handling is checked in fresh processes below. +const testTheme = Object.fromEntries( + [ + 'keyword', + 'built_in', + 'literal', + 'number', + 'regexp', + 'string', + 'comment', + 'doctag', + 'meta', + 'function', + 'title', + 'class', + 'type', + 'tag', + 'name', + 'attr', + 'attribute', + 'variable', + 'params', + 'operator', + 'addition', + 'deletion', + 'section', + 'strong', + 'emphasis', + 'code', + ].map((scope) => [scope, (text: string) => `\x1b[35m${text}\x1b[39m`]), +); + +test('highlights every supported language and preserves code units and logical lines', () => { + const edgeCases = '\n\t<&>"\' & ' 😀 e\u0301 😀 \ud800|\udfff\r\n\n '; + for (const [language, example] of Object.entries(languageExamples)) { + const highlighted = highlightMarkdownCode(example, language, testTheme); + assert.match(highlighted.join('\n'), /\x1b\[35m/u, `${language} must classify tokens`); + for (const code of [example, `${example}${edgeCases}`, '', '\n\n', ' \t\r\n ']) { + assert.deepEqual( + highlightMarkdownCode(code, language, testTheme).map(stripAnsi), + code.split('\n'), + `${language} must preserve text, whitespace, and empty lines`, + ); + } + } +}); + +test('every documented alias has the same highlighting as its canonical language', () => { + for (const [alias, language] of Object.entries(aliases)) { + const code = languageExamples[language]; + const expected = highlightMarkdownCode(code, language, testTheme); + assert.match(expected.join('\n'), /\x1b\[35m/u); + assert.deepEqual(highlightMarkdownCode(code, alias, testTheme), expected, alias); + assert.deepEqual( + highlightMarkdownCode(code, ` ${alias.toUpperCase()} `, testTheme), + expected, + `${alias} is case-insensitive and trimmed`, + ); + } +}); + +test('highlights tagged TypeScript code without changing its text', () => { + const highlightCode = markdownTheme.highlightCode; + assert.ok(highlightCode, 'the shared Markdown theme must provide the highlighting hook'); + const code = ['const greeting: string = "你好";', 'console.log(greeting);'].join('\n'); + + const highlighted = highlightCode(code, 'typescript'); + + assert.equal(stripAnsi(highlighted.join('\n')), code); +}); + +test('classifies aliased language tokens without changing escaped source text', () => { + const code = 'const label = "你好<&>";'; + const highlighted = highlightMarkdownCode(code, ' TS ', { + keyword: (text) => `${text}`, + string: (text) => `${text}`, + }).join('\n'); + + assert.match(highlighted, /const<\/keyword>/u); + assert.match(highlighted, /你好<\/string>/u); + assert.equal(highlighted.replace(/<\/?(?:keyword|string)>/gu, ''), code); +}); + +test('falls back to plain code for absent, unknown, or failed highlighting', () => { + const code = [' first line', '', '最后一行 '].join('\n'); + + assert.deepEqual(highlightMarkdownCode(code), code.split('\n')); + for (const language of [ + '', + ' ', + 'text', + 'plaintext', + 'made-up-language', + '__proto__', + 'constructor', + ]) { + assert.deepEqual(highlightMarkdownCode(code, language, testTheme), code.split('\n'), language); + } + assert.deepEqual( + highlightMarkdownCode('const value = 1;', 'typescript', { + keyword: () => { + throw new Error('theme failed'); + }, + }), + ['const value = 1;'], + ); +}); + +test('large streamed code falls back to plain text before formatting', () => { + const prefix = 'const value = 1;\n'; + for (const [language, code] of [ + ['typescript', prefix + 'x'.repeat(4097 - prefix.length)], + ['markdown', '['.repeat(2048) + 'x' + ']'.repeat(2048)], + ]) { + let formattingCalls = 0; + const countFormatting = (text: string) => { + formattingCalls++; + return text; + }; + assert.deepEqual( + highlightMarkdownCode(code, language, { + keyword: countFormatting, + link: countFormatting, + string: countFormatting, + default: countFormatting, + }), + code.split('\n'), + ); + assert.equal( + formattingCalls, + 0, + `${language} must bypass synchronous highlighting above the cap`, + ); + } +}); + +test('production highlighting respects terminal color capabilities and NO_COLOR', () => { + const code = languageExamples.typescript; + const script = [ + 'const { markdownTheme } = await import(process.argv[1]);', + 'if (!markdownTheme.highlightCode) throw new Error("missing shared Markdown hook");', + 'process.stdout.write(JSON.stringify(markdownTheme.highlightCode(process.argv[2], "typescript")));', + ].join('\n'); + const moduleUrl = new URL('../pi-transcript-format.js', import.meta.url).href; + const cases = [ + { + name: 'truecolor', + env: { TERM: 'xterm-256color', COLORTERM: 'truecolor', NO_COLOR: '' }, + colored: true, + }, + { name: 'basic ANSI', env: { TERM: 'xterm', COLORTERM: '', NO_COLOR: '' }, colored: true }, + { + name: 'NO_COLOR', + env: { TERM: 'xterm-256color', COLORTERM: 'truecolor', NO_COLOR: '1' }, + colored: false, + }, + { + name: 'dumb terminal', + env: { TERM: 'dumb', COLORTERM: 'truecolor', NO_COLOR: '' }, + colored: false, + }, + ]; + for (const { name, env, colored } of cases) { + const output = execFileSync( + process.execPath, + ['--input-type=module', '-e', script, moduleUrl, code], + { + encoding: 'utf8', + env: { ...process.env, FORCE_COLOR: '', ...env }, + timeout: 15_000, + }, + ); + const lines = JSON.parse(output) as string[]; + assert.deepEqual(lines.map(stripAnsi), code.split('\n'), name); + assert.equal(/\x1b\[[\d;]*m/u.test(lines.join('\n')), colored, name); + if (name === 'truecolor') assert.match(lines.join('\n'), /\x1b\[38;2;/u); + if (name === 'basic ANSI') assert.doesNotMatch(lines.join('\n'), /\x1b\[38;[25];/u); + } +}); + +test('highlighting preserves every live and detailed row during streaming and resizing', async (t) => { + const source = [ + 'const greeting: string = "你好<&>";', + '/* multi-line comment', + ' with e\u0301 and 😀 */', + '', + '\tconsole.log(greeting); // ' + 'long-code-'.repeat(8), + ].join('\n'); + // Load the real production styles after color capability detection, with no + // mutation of the parent process's theme or assumptions about its stdout. + const script = [ + 'const [layoutUrl, transcriptUrl, formatUrl, openText, closedText] = process.argv.slice(1);', + 'const { MakaTranscriptComponent } = await import(layoutUrl);', + 'const { createMakaPiTranscriptState } = await import(transcriptUrl);', + 'const { markdownTheme } = await import(formatUrl);', + 'const highlightCode = markdownTheme.highlightCode;', + 'if (!highlightCode) throw new Error("missing shared Markdown hook");', + 'function transcriptFor(text) {', + ' const state = createMakaPiTranscriptState();', + ' const entry = { kind: "assistant", messageId: "assistant-code", text };', + ' state.entries.push(entry);', + ' const transcript = new MakaTranscriptComponent(state, () => ({', + ' title: "Maka", cwd: "/repo", model: "model", connectionSlug: "connection", permissionMode: "ask",', + ' }));', + ' return { transcript, entry };', + '}', + 'const results = [];', + 'for (const [closed, text] of [[false, openText], [true, closedText]]) {', + ' markdownTheme.highlightCode = undefined;', + ' const baseline = transcriptFor(text);', + ' const baselineDocument = baseline.transcript.createDocumentRenderer();', + ' const before = [8, 24, 40, 100].map(width => ({', + ' width, live: baseline.transcript.render(width), detailed: baselineDocument(width).lines,', + ' }));', + ' markdownTheme.highlightCode = highlightCode;', + ' const highlighted = transcriptFor(text);', + ' const highlightedDocument = highlighted.transcript.createDocumentRenderer();', + ' for (const previous of before) results.push({', + ' closed, ...previous, afterLive: highlighted.transcript.render(previous.width),', + ' afterDetailed: highlightedDocument(previous.width).lines,', + ' sourceUnchanged: baseline.entry.text === text && highlighted.entry.text === text,', + ' });', + '}', + 'process.stdout.write(JSON.stringify(results));', + ].join('\n'); + const openText = `Code example:\n\n\`\`\`ts\n${source}`; + for (const [name, env, colored] of [ + ['truecolor', { TERM: 'xterm-256color', COLORTERM: 'truecolor', NO_COLOR: '' }, true], + ['basic ANSI', { TERM: 'xterm', COLORTERM: '', NO_COLOR: '' }, true], + ['NO_COLOR', { TERM: 'xterm-256color', COLORTERM: 'truecolor', NO_COLOR: '1' }, false], + ] as const) { + await t.test(name, () => { + const output = execFileSync( + process.execPath, + [ + '--input-type=module', + '-e', + script, + new URL('../pi-tui-layout.js', import.meta.url).href, + new URL('../pi-transcript.js', import.meta.url).href, + new URL('../pi-transcript-format.js', import.meta.url).href, + openText, + `${openText}\n\`\`\`\n\nAfter the code.`, + ], + { + encoding: 'utf8', + timeout: 15_000, + env: { + ...process.env, + FORCE_COLOR: '', + ...env, + }, + }, + ); + const results = JSON.parse(output) as Array<{ + closed: boolean; + width: number; + live: string[]; + detailed: string[]; + afterLive: string[]; + afterDetailed: string[]; + sourceUnchanged: boolean; + }>; + assert.equal(results.length, 8); + for (const { + closed, + width, + live, + detailed, + afterLive, + afterDetailed, + sourceUnchanged, + } of results) { + const context = `${closed ? 'closed' : 'streaming'} fence at width ${width}`; + assert.equal(/\x1b\[[\d;]*m/u.test(afterLive.join('\n')), colored, `live ${context}`); + assert.equal( + /\x1b\[[\d;]*m/u.test(afterDetailed.join('\n')), + colored, + `detailed ${context}`, + ); + assert.deepEqual(afterLive.map(stripAnsi), live.map(stripAnsi), `live ${context}`); + assert.deepEqual( + afterDetailed.map(stripAnsi), + detailed.map(stripAnsi), + `detailed ${context}`, + ); + assert.deepEqual(afterLive.map(stripAnsi), afterDetailed.map(stripAnsi), context); + assert.equal(sourceUnchanged, true, context); + } + }); + } +}); diff --git a/packages/cli/src/pi-transcript-format.ts b/packages/cli/src/pi-transcript-format.ts index 420b6a55b86..9b2b1c6edb3 100644 --- a/packages/cli/src/pi-transcript-format.ts +++ b/packages/cli/src/pi-transcript-format.ts @@ -28,6 +28,7 @@ import { projectAgentSwarmResult } from '@maka/core/agent-swarm'; import { ptyHumanTerminalText } from '@maka/core/pty-output-view'; import { type ShellOutput } from '@maka/core/shell-run'; import { ansi } from './tui-ansi.js'; +import { highlightMarkdownCode } from './tui-syntax-highlight.js'; export function renderIndented(text: string, width: number, indent: number): string[] { const prefix = ' '.repeat(indent); @@ -185,6 +186,7 @@ export const markdownTheme: MarkdownTheme = { code: ansi.yellow, codeBlock: (text) => text, codeBlockBorder: ansi.dim, + highlightCode: highlightMarkdownCode, quote: ansi.dim, quoteBorder: ansi.dim, hr: ansi.dim, diff --git a/packages/cli/src/tui-syntax-highlight.ts b/packages/cli/src/tui-syntax-highlight.ts new file mode 100644 index 00000000000..2587bc4f589 --- /dev/null +++ b/packages/cli/src/tui-syntax-highlight.ts @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import hljs from 'highlight.js/lib/core'; +import bash from 'highlight.js/lib/languages/bash'; +import c from 'highlight.js/lib/languages/c'; +import cpp from 'highlight.js/lib/languages/cpp'; +import csharp from 'highlight.js/lib/languages/csharp'; +import css from 'highlight.js/lib/languages/css'; +import diff from 'highlight.js/lib/languages/diff'; +import go from 'highlight.js/lib/languages/go'; +import java from 'highlight.js/lib/languages/java'; +import javascript from 'highlight.js/lib/languages/javascript'; +import json from 'highlight.js/lib/languages/json'; +import markdown from 'highlight.js/lib/languages/markdown'; +import powershell from 'highlight.js/lib/languages/powershell'; +import python from 'highlight.js/lib/languages/python'; +import rust from 'highlight.js/lib/languages/rust'; +import sql from 'highlight.js/lib/languages/sql'; +import typescript from 'highlight.js/lib/languages/typescript'; +import xml from 'highlight.js/lib/languages/xml'; +import yaml from 'highlight.js/lib/languages/yaml'; +import { ansi } from './tui-ansi.js'; + +type HighlightFormatter = (text: string) => string; + +export type TuiSyntaxHighlightTheme = Readonly>>; + +const languages = { + bash, + c, + cpp, + csharp, + css, + diff, + go, + java, + javascript, + json, + markdown, + powershell, + python, + rust, + sql, + typescript, + xml, + yaml, +}; + +for (const [name, language] of Object.entries(languages)) { + hljs.registerLanguage(name, language); +} + +/** + * Fence aliases are intentionally explicit. Unknown, blank, and plain-text + * tags stay unhighlighted instead of invoking unreliable auto-detection. + */ +const languageAliases: Readonly> = { + sh: 'bash', + shell: 'bash', + zsh: 'bash', + 'c++': 'cpp', + cs: 'csharp', + html: 'xml', + htm: 'xml', + svg: 'xml', + js: 'javascript', + jsx: 'javascript', + mjs: 'javascript', + cjs: 'javascript', + md: 'markdown', + ps1: 'powershell', + py: 'python', + rs: 'rust', + ts: 'typescript', + tsx: 'typescript', + yml: 'yaml', +}; + +const defaultTheme: TuiSyntaxHighlightTheme = { + keyword: ansi.accent, + built_in: ansi.yellow, + literal: ansi.yellow, + number: ansi.yellow, + regexp: ansi.green, + string: ansi.green, + comment: ansi.dim, + doctag: ansi.dim, + meta: ansi.muted, + function: ansi.accent, + title: ansi.accent, + class: ansi.accent, + type: ansi.accent, + tag: ansi.accent, + name: ansi.accent, + attr: ansi.yellow, + attribute: ansi.yellow, + variable: ansi.yellow, + params: ansi.muted, + operator: ansi.yellow, + addition: ansi.green, + deletion: ansi.red, +}; + +/** + * Highlight a Markdown fence into independent terminal lines. + * + * The optional theme keeps token classification testable without assuming a + * TTY. Production uses Maka's capability-aware ANSI functions, so NO_COLOR and + * basic terminals inherit the same behavior as the rest of the TUI. + */ +export function highlightMarkdownCode( + code: string, + language?: string, + theme: TuiSyntaxHighlightTheme = defaultTheme, +): string[] { + // The Markdown hook is synchronous and reruns as a fence streams. Bound its + // input to 4,096 UTF-16 code units: grammars can take superlinear time on + // repetitive input. Larger blocks retain plain text on the input path. + if (code.length > 4 * 1024) return code.split('\n'); + const normalizedLanguage = normalizeLanguage(language); + if (!normalizedLanguage) return code.split('\n'); + + try { + const html = hljs.highlight(code, { + language: normalizedLanguage, + ignoreIllegals: true, + }).value; + return renderHighlightedHtml(html, theme).split('\n'); + } catch { + // Rendering a response must not fail because a grammar or theme failed. + return code.split('\n'); + } +} + +function normalizeLanguage(language: string | undefined): string | undefined { + const requested = language?.trim().toLowerCase(); + if (!requested) return undefined; + const normalized = languageAliases[requested] ?? requested; + return Object.hasOwn(languages, normalized) ? normalized : undefined; +} + +const HTML_TOKEN = /]*>|<\/span>|&(?:amp|lt|gt|quot|apos|#(?:\d+|x[\da-f]+));/giu; + +function renderHighlightedHtml(html: string, theme: TuiSyntaxHighlightTheme): string { + const scopes: Array = []; + let output = ''; + let cursor = 0; + + for (const match of html.matchAll(HTML_TOKEN)) { + const index = match.index; + appendStyled(html.slice(cursor, index)); + const token = match[0]; + + if (token.startsWith('') { + scopes.pop(); + } else { + appendStyled(decodeHtmlEntity(token)); + } + cursor = index + token.length; + } + appendStyled(html.slice(cursor)); + return output; + + function appendStyled(text: string): void { + if (!text) return; + const formatter = activeFormatter(scopes, theme); + if (!formatter) { + output += text; + return; + } + // Keep every rendered line self-contained. A multi-line comment must not + // leave an ANSI style open across the array boundary consumed by pi-tui. + output += text + .split('\n') + .map((line) => (line ? formatter(line) : '')) + .join('\n'); + } +} + +function scopeFromSpan(tag: string): string | undefined { + const classes = /\bclass=(?:"([^"]*)"|'([^']*)')/u.exec(tag); + return (classes?.[1] ?? classes?.[2]) + ?.split(/\s+/u) + .find((className) => className.startsWith('hljs-')) + ?.slice('hljs-'.length); +} + +function activeFormatter( + scopes: readonly (string | undefined)[], + theme: TuiSyntaxHighlightTheme, +): HighlightFormatter | undefined { + for (let index = scopes.length - 1; index >= 0; index -= 1) { + const scope = scopes[index]; + if (!scope) continue; + const [prefix] = scope.split(/[.-]/u); + const formatter = theme[scope] ?? (prefix ? theme[prefix] : undefined); + if (formatter) return formatter; + } + return theme.default; +} + +function decodeHtmlEntity(entity: string): string { + const body = entity.slice(1, -1); + switch (body) { + case 'amp': + return '&'; + case 'lt': + return '<'; + case 'gt': + return '>'; + case 'quot': + return '"'; + case 'apos': + return "'"; + default: { + const radix = body.slice(0, 2).toLowerCase() === '#x' ? 16 : 10; + const digits = body.slice(radix === 16 ? 2 : 1); + const codePoint = Number.parseInt(digits, radix); + return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff + ? String.fromCodePoint(codePoint) + : entity; + } + } +} diff --git a/patches/@earendil-works+pi-tui+0.87.1.patch b/patches/@earendil-works+pi-tui+0.87.1.patch index a4ceefa673d..89fd4b87cd5 100644 --- a/patches/@earendil-works+pi-tui+0.87.1.patch +++ b/patches/@earendil-works+pi-tui+0.87.1.patch @@ -1,5 +1,5 @@ diff --git a/node_modules/@earendil-works/pi-tui/dist/components/editor.js b/node_modules/@earendil-works/pi-tui/dist/components/editor.js -index 7eae102..9d00340 100644 +index 7eae102..bcdcf6c 100644 --- a/node_modules/@earendil-works/pi-tui/dist/components/editor.js +++ b/node_modules/@earendil-works/pi-tui/dist/components/editor.js @@ -2,7 +2,6 @@ import { getKeybindings } from "../keybindings.js"; @@ -43,3 +43,29 @@ index 7eae102..9d00340 100644 } undo() { this.exitHistoryBrowsing(); +diff --git a/node_modules/@earendil-works/pi-tui/dist/utils.js b/node_modules/@earendil-works/pi-tui/dist/utils.js +index 31bb2e1..484c89e 100644 +--- a/node_modules/@earendil-works/pi-tui/dist/utils.js ++++ b/node_modules/@earendil-works/pi-tui/dist/utils.js +@@ -803,10 +803,10 @@ function wrapSingleLine(line, width) { + let currentVisibleLength = 0; + for (const token of tokens) { + const tokenVisibleLength = visibleWidth(token); +- const isWhitespace = token.trim() === ""; ++ const isWhitespace = stripTerminalSequences(token).trim() === ""; + // Token itself is too long - break it character by character + if (tokenVisibleLength > width && !isWhitespace) { +- if (currentLine) { ++ if (stripTerminalSequences(currentLine)) { + // Add specific reset for underline only (preserves background) + const lineEndReset = tracker.getLineEndReset(); + if (lineEndReset) { +@@ -852,7 +852,7 @@ function wrapSingleLine(line, width) { + } + updateTrackerFromText(token, tracker); + } +- if (currentLine) { ++ if (stripTerminalSequences(currentLine)) { + // No reset at end of final line - let caller handle it + wrapped.push(currentLine); + } diff --git a/scripts/check-tui-copy.mjs b/scripts/check-tui-copy.mjs index e03e5f6f41a..98f89a34a63 100644 --- a/scripts/check-tui-copy.mjs +++ b/scripts/check-tui-copy.mjs @@ -58,6 +58,7 @@ export const EXCLUDED_TUI_FILES = [ 'packages/cli/src/tui-editor-render.ts', 'packages/cli/src/tui-mcp-control.ts', 'packages/cli/src/tui-mcp-remote-publication.ts', + 'packages/cli/src/tui-syntax-highlight.ts', ]; const CJK = /[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/u; diff --git a/scripts/release-cli-file-policy.mjs b/scripts/release-cli-file-policy.mjs index d29c5805b19..9ef7e241a9a 100644 --- a/scripts/release-cli-file-policy.mjs +++ b/scripts/release-cli-file-policy.mjs @@ -211,6 +211,51 @@ export function releaseNpmEnvironment(environment, userConfigPath) { }; } +export const CLI_HIGHLIGHT_JS_LANGUAGE_MODULES = Object.freeze([ + 'bash', + 'c', + 'cpp', + 'csharp', + 'css', + 'diff', + 'go', + 'java', + 'javascript', + 'json', + 'markdown', + 'powershell', + 'python', + 'rust', + 'sql', + 'typescript', + 'xml', + 'yaml', +]); + +const HIGHLIGHT_JS_RUNTIME_DIRECTORIES = new Set(['es', 'es/languages', 'lib']); +const HIGHLIGHT_JS_RUNTIME_FILES = new Set([ + 'LICENSE', + 'package.json', + 'es/core.js', + 'es/package.json', + 'lib/core.js', + ...CLI_HIGHLIGHT_JS_LANGUAGE_MODULES.map((language) => `es/languages/${language}.js`), +]); + +/** + * Keep the CLI release's syntax highlighter dependency to its actual ESM + * runtime closure. npm publishes every highlight.js grammar and theme in one + * package, while Maka deliberately registers a bounded language set. + */ +export function isCliThirdPartyReleasePath(packageKey, relativePath) { + if (packageKey !== 'highlight.js@11.12.0') return true; + const portablePath = relativePath.split(/[\\/]/u).filter(Boolean).join('/'); + return ( + HIGHLIGHT_JS_RUNTIME_DIRECTORIES.has(portablePath) || + HIGHLIGHT_JS_RUNTIME_FILES.has(portablePath) + ); +} + export function isThirdPartyDevelopmentArtifact(relativePath) { const segments = relativePath.split(/[\\/]/).filter(Boolean); if (segments.some((segment) => DEVELOPMENT_DIRECTORIES.has(segment))) return true; diff --git a/scripts/release-cli-file-policy.test.mjs b/scripts/release-cli-file-policy.test.mjs index 0cbf0c401ff..63ec396d807 100644 --- a/scripts/release-cli-file-policy.test.mjs +++ b/scripts/release-cli-file-policy.test.mjs @@ -31,7 +31,9 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { describe, test } from 'node:test'; import { + CLI_HIGHLIGHT_JS_LANGUAGE_MODULES, collectWorkspaceDependencyClosure, + isCliThirdPartyReleasePath, isCurrentDevelopmentJavaScript, isMakaDevelopmentArtifact, isThirdPartyDevelopmentArtifact, @@ -208,6 +210,43 @@ describe('CLI release file policy', () => { } }); + test('keeps only the reviewed highlight.js runtime closure in CLI artifacts', () => { + for (const path of [ + 'package.json', + 'LICENSE', + 'es', + 'es/core.js', + 'es/package.json', + 'es/languages', + 'es/languages/typescript.js', + 'lib', + 'lib/core.js', + ]) { + assert.equal(isCliThirdPartyReleasePath('highlight.js@11.12.0', path), true, path); + } + for (const path of [ + 'README.md', + 'es/index.js', + 'es/languages/ada.js', + 'lib/languages/typescript.js', + 'styles/github.css', + 'types/index.d.ts', + ]) { + assert.equal(isCliThirdPartyReleasePath('highlight.js@11.12.0', path), false, path); + } + + assert.equal(isCliThirdPartyReleasePath('another-package@1.0.0', 'README.md'), true); + + const highlighterSource = readFileSync( + resolve(import.meta.dirname, '../packages/cli/src/tui-syntax-highlight.ts'), + 'utf8', + ); + const importedLanguages = [ + ...highlighterSource.matchAll(/from 'highlight\.js\/lib\/languages\/([^']+)'/gu), + ].map((match) => match[1]); + assert.deepEqual(importedLanguages.sort(), [...CLI_HIGHLIGHT_JS_LANGUAGE_MODULES].sort()); + }); + test('keeps the stricter Maka-owned package boundary', () => { assert.equal(isMakaDevelopmentArtifact('src/index.js'), true); assert.equal(isMakaDevelopmentArtifact('dist/__tests__/fixture.js'), true); diff --git a/scripts/release-cli-package.mjs b/scripts/release-cli-package.mjs index fb4aaacaf83..2344d9250c2 100644 --- a/scripts/release-cli-package.mjs +++ b/scripts/release-cli-package.mjs @@ -41,6 +41,7 @@ import { npmSpawnOptions } from './npm-spawn.mjs'; import { assertProductNightlyVersion } from './release-version.mjs'; import { isCurrentDevelopmentJavaScript, + isCliThirdPartyReleasePath, isMakaDevelopmentArtifact, isThirdPartyDevelopmentArtifact, orderWorkspaceBuilds, @@ -453,9 +454,12 @@ function copyRuntimeDist(source, destination, packageName = 'maka-agent') { } function copyThirdPartyPackage(source, destination) { - if (!existsSync(join(source, 'package.json'))) { + const manifestPath = join(source, 'package.json'); + if (!existsSync(manifestPath)) { throw new Error(`Installed dependency has no package.json: ${source}`); } + const manifest = readJson(manifestPath); + const packageKey = `${manifest.name}@${manifest.version}`; rmSync(destination, { recursive: true, force: true }); cpSync(source, destination, { recursive: true, @@ -465,7 +469,8 @@ function copyThirdPartyPackage(source, destination) { const relativePath = relative(source, path); return ( !relativePath.split(sep).includes('node_modules') && - !isThirdPartyDevelopmentArtifact(relativePath) + !isThirdPartyDevelopmentArtifact(relativePath) && + isCliThirdPartyReleasePath(packageKey, relativePath) ); }, });