From da3b47ad41482673015db86774844605327654e9 Mon Sep 17 00:00:00 2001 From: Sun-GLiang <81428141+Sun-GLiang@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:45:25 +0800 Subject: [PATCH 01/13] refactor(media): share session image rendering across native and ACP output Route external media through the Host artifact pipeline and reuse bounded image/SVG previews across chat, tools and artifacts. Bump the compatibility epoch for mixed-content tool results. Generated-by: Codex --- .../licenses/npm/THIRD_PARTY_NOTICES.txt | 17 ++ .../artifact-preview-registry.test.ts | 6 +- .../artifact-preview-registry-shell.tsx | 23 ++- .../session-media-preview.zh-CN.md | 64 +++++++ package-lock.json | 2 +- .../src/__tests__/acp-executor-plugin.test.ts | 55 ++++++ .../src/__tests__/acp-projection.test.ts | 50 +++++ .../acp-executor-plugin/src/acp-projection.ts | 82 ++++++++- packages/acp-executor-plugin/src/index.ts | 91 ++++++++- packages/cli/THIRD_PARTY_NOTICES.txt | 17 ++ packages/cli/src/pi-transcript-format.ts | 2 + packages/core/package.json | 5 +- .../core/src/__tests__/session-media.test.ts | 125 +++++++++++++ packages/core/src/artifacts.ts | 17 +- packages/core/src/events.ts | 1 + packages/core/src/session-media.ts | 104 +++++++++++ packages/core/src/svg-preview.ts | 174 ++++++++++++++++++ .../core/src/tool-result-record-schema.ts | 9 + .../src/__tests__/chat-image-delivery.test.ts | 77 +++++++- packages/runtime-host/src/protocol/index.ts | 4 +- .../src/server/chat-image-delivery.ts | 90 ++++++++- .../src/server/chat-image-source.ts | 2 +- .../src/server/execution-composition.ts | 1 + .../src/__tests__/conversation-copy.test.ts | 54 ++++++ .../__tests__/plugin-executor-backend.test.ts | 52 ++++++ packages/runtime/src/conversation-copy.ts | 18 ++ .../runtime/src/filesystem-worker/protocol.ts | 2 +- packages/runtime/src/image-file.ts | 16 +- .../runtime/src/plugin-executor-backend.ts | 20 +- .../runtime/src/plugin-executor-service.ts | 60 +++++- packages/runtime/src/tool-runtime.ts | 1 + packages/storage/src/artifact-store.ts | 18 +- .../ui/scripts/benchmark-session-media.mjs | 65 +++++++ .../scripts/markdown-images.browser.test.mjs | 48 ++++- .../src/__tests__/attachment-image.test.tsx | 34 ++++ .../ui/src/__tests__/markdown-body.test.ts | 15 ++ packages/ui/src/artifact-preview-registry.ts | 4 +- packages/ui/src/attachment-image.tsx | 28 ++- packages/ui/src/index.ts | 4 + packages/ui/src/markdown-body.tsx | 45 +++-- packages/ui/src/markdown-image.tsx | 62 +------ packages/ui/src/media-image-source.ts | 52 ++++++ packages/ui/src/session-media-image.tsx | 78 ++++++++ packages/ui/src/shared-ui-copy.ts | 12 ++ packages/ui/src/svg-code-preview.tsx | 65 +++++++ .../src/tool-activity/tool-result-preview.tsx | 10 + 46 files changed, 1636 insertions(+), 145 deletions(-) create mode 100644 docs/architecture/session-media-preview.zh-CN.md create mode 100644 packages/core/src/__tests__/session-media.test.ts create mode 100644 packages/core/src/session-media.ts create mode 100644 packages/core/src/svg-preview.ts create mode 100644 packages/ui/scripts/benchmark-session-media.mjs create mode 100644 packages/ui/src/media-image-source.ts create mode 100644 packages/ui/src/session-media-image.tsx create mode 100644 packages/ui/src/svg-code-preview.tsx diff --git a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt index dbe751bc9b9..3f0de534f84 100644 --- a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt +++ b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt @@ -5828,6 +5828,23 @@ See the LICENSE file in the root of this repository. ================================================================================ +Package: @xmldom/xmldom@0.8.15 +Declared license: MIT +Selected license: MIT +Repository: git://github.com/xmldom/xmldom.git + +--- LICENSE --- +Copyright 2019 - present Christopher J. Brody and other contributors, as listed in: https://github.com/xmldom/xmldom/graphs/contributors +Copyright 2012 - 2017 @jindw and other contributors, as listed in: https://github.com/jindw/xmldom/graphs/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + Package: @xterm/addon-fit@0.11.0 Declared license: MIT Selected license: MIT diff --git a/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts b/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts index 4ae9aa4b6ed..87f5e60ffd0 100644 --- a/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts +++ b/apps/desktop/src/main/__tests__/artifact-preview-registry.test.ts @@ -31,7 +31,7 @@ import { describe('artifact preview registry', () => { it('treats present MIME metadata as authoritative and rejects unsafe formats', () => { assert.deepEqual( - resolvePreviewKind({ name: 'tricky.png', kind: 'image', mimeType: 'image/svg+xml' }), + resolvePreviewKind({ name: 'tricky.png', kind: 'image', mimeType: 'text/html' }), { kind: 'unsupported', reason: 'mime_disallowed' }, ); }); @@ -77,11 +77,11 @@ describe('artifact preview registry', () => { { ok: true, base64: 'A'.repeat(3 * 1024 * 1024), - mimeType: 'image/svg+xml', + mimeType: 'text/html', }, 'oversize', ], - [{ ok: true, base64: 'AAAA', mimeType: 'image/svg+xml' }, 'mime_disallowed'], + [{ ok: true, base64: 'AAAA', mimeType: 'text/html' }, 'mime_disallowed'], ]; for (const [result, reason] of rejected) { const outcome = decideImageReadOutcome(result); diff --git a/apps/desktop/src/renderer/features/workbar/tools/artifacts/artifact-preview-registry-shell.tsx b/apps/desktop/src/renderer/features/workbar/tools/artifacts/artifact-preview-registry-shell.tsx index bad400ef134..c5fa56d7143 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/artifacts/artifact-preview-registry-shell.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/artifacts/artifact-preview-registry-shell.tsx @@ -17,11 +17,11 @@ * under the License. */ -/** Renderer shell for capped raster previews and typed unsupported outcomes. */ +/** Renderer shell for bounded image previews and typed unsupported outcomes. */ import { useEffect, useState } from 'react'; import type { ArtifactDescriptor } from '@maka/core/artifacts'; -import { Button, useUiLocale } from '@maka/ui'; +import { Button, useUiLocale, mediaImageSource, MediaImageDisplay } from '@maka/ui'; import { Banner } from '@astryxdesign/core/Banner'; import { Spinner } from '@astryxdesign/core/Spinner'; import { @@ -152,7 +152,7 @@ export function UnsupportedArtifactPreview(props: { * - The post-load L2 decision (`decideImageReadOutcome`) runs * INSIDE the async, BEFORE `setState`. The hook's state union * therefore holds either a `loading` placeholder, an `image` - * branch carrying the verified base64, OR an `unsupported` + * branch carrying the verified image source, OR an `unsupported` * branch that carries NO base64. A 10MB oversize payload never * enters React state / DevTools snapshot. * - The component below is intentionally a thin switch over the @@ -190,10 +190,7 @@ function ImageArtifactPreview(props: { // result.state === 'image' — exhaustiveness check below. return (
- {props.record.name} +
); } @@ -216,9 +213,8 @@ function toPreviewInput(record: ArtifactDescriptor): ArtifactPreviewInput { * branch). That meant a 10MB oversize payload could land in React * state / DevTools before the L2 cap rejected it for rendering. The * fix replaces the broad union with this closed three-way one and - * runs the L2 decision INSIDE the async, so the only `base64` that - * ever reaches `setState` is one that has already passed - * `decideImageReadOutcome` (cap + MIME re-validation). + * runs the L2 decision INSIDE the async. The source passed to state + * has passed MIME/size validation and, for SVG, static preview conversion. * * Source/test gate (locked by `artifact-preview-registry.test.ts`): * the `unsupported` branch type signature MUST NOT include `base64`. @@ -228,7 +224,7 @@ function toPreviewInput(record: ArtifactDescriptor): ArtifactPreviewInput { */ type ImagePreviewLoadState = | { state: 'loading' } - | { state: 'image'; safeMime: string; base64: string } + | { state: 'image'; src: string; onError(): void } | { state: 'unsupported'; reason: Extract['reason']; @@ -254,7 +250,10 @@ function useImagePreviewLoad( // receives the post-decision shape. const outcome = decideImageReadOutcome(raw); if (outcome.kind === 'image') { - setState({ state: 'image', safeMime: outcome.safeMime, base64: outcome.base64 }); + const src = await mediaImageSource(raw); + if (cancelled) return; + if (!src) { setState({ state: 'unsupported', reason: 'read_failed' }); return; } + setState({ state: 'image', src, onError: () => setState({ state: 'unsupported', reason: 'read_failed' }) }); } else { setState({ state: 'unsupported', reason: outcome.reason }); } diff --git a/docs/architecture/session-media-preview.zh-CN.md b/docs/architecture/session-media-preview.zh-CN.md new file mode 100644 index 00000000000..e8f43163e36 --- /dev/null +++ b/docs/architecture/session-media-preview.zh-CN.md @@ -0,0 +1,64 @@ + + +# 会话媒体接入与预览 + +## 分层 + +- `core/session-media` 定义执行器无关的输入（内联字节、来源地址）与有序内容项（文本、已保存图片、diff）。图片输出只携带当前 Session 的 Artifact 引用，不把 base64 放进消息事件。 +- `ChatImageDeliveryService` 同时接收已完成正文中的 Markdown 图片和执行器的 `publish` 请求，复用读取边界、联网策略、并发队列、校验、存储配额、去重和 Artifact 保存。 +- `PluginExecutorContext.publishMedia` 将外部执行器的输入交给 Host；ACP 适配层处理 image、resource_link、embedded resource，按到达顺序等待发布结果。 +- `SessionMediaImage` 是正文与工具结果的公共组件。用户附件沿用同一字节读取器；产物面板复用同一图片源转换器和最终图片组件。 + +## 消息与兼容 + +正文中的结构化图片转换为 `maka://runtime/attachments/` Markdown 引用，沿用现有文本事件和历史格式，保留文字与图片顺序。工具结果新增 `content.parts`，允许文本、图片和 diff 混排。旧的 text、file_diff、image 结果保持可读。 + +混合工具结果属于新的 wire 内容类型，Runtime Host 兼容性 epoch 从 227 升为 228，避免旧客户端接收无法解码的结果。合并前需对照 main 重新确认 epoch。 + +复制会话时收集并重写混合内容里的图片引用。CLI 将混合内容按顺序输出为文本描述。没有媒体发布能力的执行器宿主显示明确的不可用描述；发布失败不会将二进制内容倾倒进正文。 + +文件写入操作本身不等于发布图片。生成的文件仍需通过 Markdown 图片引用或结构化媒体返回，才进入正文或工具结果预览。 + +## SVG + +本地文件、下载内容和 ACP 内嵌 SVG 共用格式校验，原始文件保存为会话 Artifact。显示时从原始内容生成静态 SVG 预览，并通过图片元素加载，不注入应用 DOM。脚本、事件处理器、外部资源、动画、foreignObject、use 等不进入预览；允许基本几何、文本、渐变、受限行内样式和静态 class 样式规则。 + +完整的 `svg` 代码块在流式输出结束后提供预览、源码切换及静态预览下载。代码块的原始源码由现有消息历史保存，下载按钮下载经过处理的 SVG。原文件仍保存在会话 Artifact 中，不改变现有产物列表的可见性规则。 + +显示支持与模型输入支持分开：SVG 不自动成为模型的视觉输入。现有 raster MIME 校验继续用于模型输入。 + +## 性能约束 + +- 普通文本不导入 SVG XML 解析器；前端解析模块使用动态 import。 +- 图片和 SVG 代码块在接近视口时才加载；流式 SVG 代码块只显示源码。 +- 单条正文自动预览最多 3 个 SVG，每个自动预览源码最多 64 Ki 字符，总计最多 128 Ki 字符，超出后显示显式预览按钮。 +- SVG 最大 256 Ki 字符、4,000 个保留元素、64 层深度。 +- 会话图片读取合并并发请求；已读取图片缓存最多 32 项、8 Mi 字符，显式重试会重新读取。 +- SVG 转换缓存最多 16 项、1 Mi 字符，重复消费复用转换结果。 +- Host 复用原有最多 2 个并行捕获、128 个排队任务的限制；排队内联字节最多 8 MiB，单图片最多 2 MiB。 +- ACP 限制每轮保留的媒体数量与数据总量；媒体发布失败或超限转成有界文本提示。 + +## 验证入口 + +核心测试覆盖输入边界、SVG 净化和模型输入隔离；Host 测试覆盖重复发布、真实 MIME、原文件删除后回放、配额和联网失败；ACP 测试覆盖异步发布与混排顺序；UI 测试覆盖组件共用和重新挂载时的缓存读取。浏览器回归位于 `packages/ui/scripts/markdown-images.browser.test.mjs`，可用 `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` 指定本地 Chromium。 + +## 局部性能基准 + +`node packages/ui/scripts/benchmark-session-media.mjs [git-ref]` 比较改动前后的正文图表预算扫描函数。每种输入运行 7 轮、每轮 3,000 次，取中位数。2026-10-22 本机对改动前 HEAD 的结果：普通正文 14.74 → 3.70 ms、图片 Markdown 4.73 → 0.50 ms、普通代码块 17.61 → 0.27 ms、Mermaid 5.91 → 6.10 ms。普通消息通过快速路径减少了分行分配；Mermaid 的每次差异约 0.06 微秒。此结果仅覆盖该函数，不代表真实长会话滚动帧率或应用整体启动性能。 diff --git a/package-lock.json b/package-lock.json index a35cc9687ad..e576202aae1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6340,7 +6340,6 @@ "version": "0.8.15", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.15.tgz", "integrity": "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==", - "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" @@ -17291,6 +17290,7 @@ "version": "0.1.0", "license": "Apache-2.0", "dependencies": { + "@xmldom/xmldom": "^0.8.15", "intl-messageformat": "11.2.15", "marked": "18.0.11" } diff --git a/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts b/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts index 538c91f3ccd..c6f5f555b5b 100644 --- a/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts +++ b/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts @@ -1975,6 +1975,61 @@ test('questions retain option identity and output updates retain arrival order', } }); +test('ACP waits for asynchronous media publication and preserves surrounding text', async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + const events: unknown[] = []; + try { + const result = await executor.execute(request('media'), { + ...executorContext(events), + requestPermission: async () => { + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'before' }, + }); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'image', mimeType: 'image/png', data: 'AAAA' }, + }); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'after' }, + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return { + kind: 'image', + mimeType: 'image/png', + ref: { kind: 'session_file', sessionId: 'session-a', relativePath: 'saved-image' }, + }; + }, + }); + assert.equal(result.status, 'completed'); + if (result.status !== 'completed') throw new Error('Expected completion'); + assert.equal( + result.text, + 'before\n\n![](maka://runtime/attachments/saved-image)\n\nafterreply:media', + ); + assert.equal( + events + .filter((e: any) => e.type === 'output_delta') + .map((e: any) => e.text) + .join(''), + result.text, + ); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); + function request(text: string) { return { sessionId: 'session-a', diff --git a/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts b/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts index 1833a7dfa47..c46c882b8a1 100644 --- a/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts +++ b/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts @@ -49,3 +49,53 @@ for (const { name, oldText, newText, expected } of [ ); }); } + +test('rich ACP content preserves text, SVG resource, image and diff order', async () => { + const { projectRichToolResult } = await import('../acp-projection.js'); + const inputs: unknown[] = []; + const result = await projectRichToolResult( + [ + { type: 'content', content: { type: 'text', text: 'before' } }, + { + type: 'content', + content: { + type: 'resource', + resource: { uri: 'drawing.svg', mimeType: 'image/svg+xml', text: '' }, + }, + }, + { type: 'content', content: { type: 'image', mimeType: 'image/png', data: 'AAAA' } }, + { type: 'diff', path: 'drawing.svg', oldText: '', newText: '' }, + { type: 'content', content: { type: 'text', text: 'after' } }, + ], + undefined, + { + signal: new AbortController().signal, + emit() {}, + requestPermission: async () => ({ outcome: 'cancelled' }), + publishMedia: async (input) => { + inputs.push(input); + return { + kind: 'image', + mimeType: input.kind === 'inline' ? input.mimeType : 'image/png', + ref: { + kind: 'session_file', + sessionId: 'session', + relativePath: `image-${inputs.length}`, + }, + }; + }, + }, + ); + assert.equal(result.kind, 'content'); + if (result.kind !== 'content') throw new Error('Expected content'); + assert.deepEqual( + result.parts.map((part) => part.kind), + ['text', 'image', 'image', 'file_diff', 'text'], + ); + assert.equal(inputs.length, 2); + assert.deepEqual(inputs[0], { + kind: 'inline', + mimeType: 'image/svg+xml', + base64: Buffer.from('').toString('base64'), + }); +}); diff --git a/packages/acp-executor-plugin/src/acp-projection.ts b/packages/acp-executor-plugin/src/acp-projection.ts index 43f73fe96ba..128823de642 100644 --- a/packages/acp-executor-plugin/src/acp-projection.ts +++ b/packages/acp-executor-plugin/src/acp-projection.ts @@ -17,13 +17,20 @@ * under the License. */ -import type { ToolCall, ToolCallContent } from '@agentclientprotocol/sdk'; +import type { ContentBlock, ToolCall, ToolCallContent } from '@agentclientprotocol/sdk'; import type { PluginExecutorContext, PluginExecutorRequest, PluginExecutorToolResultContent, } from '@maka/runtime/plugin-executor-service'; +import { + type SessionContentPart, + type SessionMediaInput, + SESSION_CONTENT_MAX_PARTS, + SESSION_MEDIA_MAX_BYTES, +} from '@maka/core/session-media'; + const MAX_EVENT_TEXT = 8_192; const MAX_TOOL_RESULT_DIFF = 1024 * 1024; @@ -166,3 +173,76 @@ export function activityKind(kind: ToolCall['kind'] | undefined) { if (kind === 'think') return 'explore' as const; return 'tool' as const; } + +/** Preserve protocol content order; only the Host can publish media bytes or read a source. */ +export async function projectContentBlock( + content: ContentBlock, + context: PluginExecutorContext, +): Promise> { + if (content.type === 'text') return { kind: 'text', text: boundedText(content.text) }; + let input: SessionMediaInput | undefined; + if (content.type === 'image') + input = { kind: 'inline', mimeType: content.mimeType, base64: content.data }; + if (content.type === 'resource_link') { + if ( + content.mimeType?.startsWith('image/') || + /\.(svg|png|jpe?g|webp|gif)(?:[?#]|$)/i.test(content.uri) + ) + input = { kind: 'source', source: content.uri }; + else return { kind: 'text', text: boundedText(`${content.name}: ${content.uri}`) }; + } + if (content.type === 'resource') { + const resource = content.resource; + if ('text' in resource) { + if (resource.mimeType === 'image/svg+xml' || /\.svg(?:[?#]|$)/i.test(resource.uri)) { + if (resource.text.length > SESSION_MEDIA_MAX_BYTES) + return { kind: 'text', text: '[Media unavailable: too large]' }; + input = { + kind: 'inline', + mimeType: 'image/svg+xml', + base64: Buffer.from(resource.text).toString('base64'), + }; + } else return { kind: 'text', text: boundedText(resource.text) }; + } else if (resource.mimeType?.startsWith('image/')) + input = { kind: 'inline', mimeType: resource.mimeType, base64: resource.blob }; + } + if (!input || !context.publishMedia) return { kind: 'text', text: '[Media preview unavailable]' }; + try { + const part = await context.publishMedia(input); + return part.kind === 'file_diff' ? { kind: 'text', text: '[Media preview unavailable]' } : part; + } catch { + return { kind: 'text', text: '[Media publication failed]' }; + } +} + +export async function projectRichToolResult( + content: readonly ToolCallContent[], + rawOutput: unknown, + context: PluginExecutorContext, +): Promise { + if (!content.some((item) => item.type === 'content' && item.content.type !== 'text')) + return projectToolResult(content, rawOutput); + const parts: SessionContentPart[] = []; + let budget = MAX_TOOL_RESULT_DIFF; + for (const item of content.slice(0, SESSION_CONTENT_MAX_PARTS)) { + const part = + item.type === 'content' + ? await projectContentBlock(item.content, context) + : item.type === 'diff' + ? projectToolResult([item], undefined) + : { kind: 'text' as const, text: boundedText(`Terminal ${item.terminalId}`) }; + if (part.kind === 'content') continue; + const normalized: SessionContentPart = + part.kind === 'file_diff' ? { ...part, paths: [...part.paths] } : part; + budget -= Buffer.byteLength(JSON.stringify(normalized), 'utf8'); + if ( + budget < 1024 || + (parts.length === SESSION_CONTENT_MAX_PARTS - 1 && content.length > SESSION_CONTENT_MAX_PARTS) + ) { + parts.push({ kind: 'text', text: '[Additional tool content omitted]' }); + break; + } + parts.push(normalized); + } + return { kind: 'content', parts }; +} diff --git a/packages/acp-executor-plugin/src/index.ts b/packages/acp-executor-plugin/src/index.ts index 802ba4af15c..14a50c9cba4 100644 --- a/packages/acp-executor-plugin/src/index.ts +++ b/packages/acp-executor-plugin/src/index.ts @@ -17,6 +17,7 @@ * under the License. */ +import { sessionContentMarkdown, SESSION_MEDIA_MAX_BASE64 } from '@maka/core/session-media'; import { createHash } from 'node:crypto'; import { constants, createReadStream } from 'node:fs'; import { access, mkdir, mkdtemp, realpath, rm, stat } from 'node:fs/promises'; @@ -53,7 +54,8 @@ import { boundedText, emitText, emitToolOutput, - projectToolResult, + projectRichToolResult, + projectContentBlock, promptText, summarizeToolContent, toolName, @@ -134,6 +136,10 @@ interface ActivePrompt { readonly context: PluginExecutorContext; readonly tools: Map; text: string; + updates: Promise; + updateError?: unknown; + mediaCount: number; + mediaBytes: number; } interface ToolSnapshot { @@ -443,7 +449,14 @@ export class AcpExecutor implements PluginExecutorProvider { } if (session.active || session.configuring || session.awaitingAck) return failure(`${this.displayName} ACP Session is busy`, 'acp_busy', true); - const active: ActivePrompt = { context, tools: new Map(), text: '' }; + const active: ActivePrompt = { + context, + tools: new Map(), + text: '', + updates: Promise.resolve(), + mediaCount: 0, + mediaBytes: 0, + }; session.active = active; try { await this.#ensureInitialized(session, context.signal); @@ -456,6 +469,8 @@ export class AcpExecutor implements PluginExecutorProvider { prompt: [{ type: 'text', text: promptText(request) }], }); const response = await this.#awaitPrompt(session, prompt, context.signal); + await active.updates; + if (active.updateError) throw active.updateError; if (!response) return { status: 'cancelled', reason: 'timeout' }; // A settled prompt can be checkpointed even when it was cancelled or failed. // Timeouts and transport failures never reach this point, and the Runtime @@ -498,6 +513,7 @@ export class AcpExecutor implements PluginExecutorProvider { } return failure(safeErrorMessage(error), errorCode(error), retryable); } finally { + await active.updates; if (session.active === active) session.active = undefined; } } @@ -974,7 +990,58 @@ export class AcpExecutor implements PluginExecutorProvider { #configureClient(session: RetainedSession, app: ClientApp): void { app .onNotification(methods.client.session.update, ({ params }) => { - if (params.sessionId === session.acpSessionId) this.#acceptUpdate(session, params.update); + if (params.sessionId !== session.acpSessionId) return; + const active = session.active; + if ( + !active || + session.restoring || + params.update.sessionUpdate === 'config_option_update' + ) { + void this.#acceptUpdate(session, params.update); + return; + } + // Bound retained protocol payloads before queuing behind an asynchronous publication. + const reserve = ( + block: import('@agentclientprotocol/sdk').ContentBlock, + ): import('@agentclientprotocol/sdk').ContentBlock => { + if (block.type === 'text') return block; + const size = + block.type === 'image' + ? block.data.length + : block.type === 'resource' + ? 'blob' in block.resource + ? block.resource.blob.length + : block.resource.text.length + : 0; + if ( + ++active.mediaCount > 64 || + size > SESSION_MEDIA_MAX_BASE64 || + active.mediaBytes + size > 8 * 1024 * 1024 + ) + return { type: 'text', text: '[Media omitted: response media limit reached]' }; + active.mediaBytes += size; + return block; + }; + let update = params.update; + if (update.sessionUpdate === 'agent_message_chunk') + update = { ...update, content: reserve(update.content) }; + else if ( + (update.sessionUpdate === 'tool_call' || update.sessionUpdate === 'tool_call_update') && + update.content + ) + update = { + ...update, + content: update.content + .slice(0, 64) + .map((item) => + item.type === 'content' ? { ...item, content: reserve(item.content) } : item, + ), + }; + active.updates = active.updates + .then(() => this.#acceptUpdate(session, update)) + .catch((error) => { + active.updateError ??= error; + }); }) .onRequest(methods.client.fs.readTextFile, async ({ params }) => { this.#assertWorkspaceAccess(session, params.sessionId); @@ -1007,7 +1074,7 @@ export class AcpExecutor implements PluginExecutorProvider { return { outcome }; } - #acceptUpdate(session: RetainedSession, update: SessionUpdate): void { + async #acceptUpdate(session: RetainedSession, update: SessionUpdate): Promise { if (session.restoring) { if (session.replay) { session.replay.updates++; @@ -1032,12 +1099,20 @@ export class AcpExecutor implements PluginExecutorProvider { emitText(active.context, 'output_delta', update.content.text); return; } + if (update.sessionUpdate === 'agent_message_chunk') { + const text = sessionContentMarkdown( + await projectContentBlock(update.content, active.context), + ); + active.text += text; + emitText(active.context, 'output_delta', text); + return; + } if (update.sessionUpdate === 'agent_thought_chunk' && update.content.type === 'text') { emitText(active.context, 'thinking_delta', update.content.text); return; } - if (update.sessionUpdate === 'tool_call') this.#acceptTool(active, update); - if (update.sessionUpdate === 'tool_call_update') this.#acceptTool(active, update); + if (update.sessionUpdate === 'tool_call') await this.#acceptTool(active, update); + if (update.sessionUpdate === 'tool_call_update') await this.#acceptTool(active, update); } #assertWorkspaceAccess(session: RetainedSession, sessionId: string): void { @@ -1050,7 +1125,7 @@ export class AcpExecutor implements PluginExecutorProvider { } } - #acceptTool(active: ActivePrompt, update: ToolCall | ToolCallUpdate): void { + async #acceptTool(active: ActivePrompt, update: ToolCall | ToolCallUpdate): Promise { const snapshot = active.tools.get(update.toolCallId) ?? { id: update.toolCallId, title: 'External tool', @@ -1098,7 +1173,7 @@ export class AcpExecutor implements PluginExecutorProvider { active.context.emit({ type: 'tool_result', toolCallId: snapshot.id, - content: projectToolResult(snapshot.content, snapshot.rawOutput), + content: await projectRichToolResult(snapshot.content, snapshot.rawOutput, active.context), ...(snapshot.status === 'failed' ? { isError: true } : {}), }); snapshot.terminal = true; diff --git a/packages/cli/THIRD_PARTY_NOTICES.txt b/packages/cli/THIRD_PARTY_NOTICES.txt index c127e54002d..7ec6e53befa 100644 --- a/packages/cli/THIRD_PARTY_NOTICES.txt +++ b/packages/cli/THIRD_PARTY_NOTICES.txt @@ -2065,6 +2065,23 @@ See the LICENSE file in the root of this repository. ================================================================================ +Package: @xmldom/xmldom@0.8.15 +Declared license: MIT +Selected license: MIT +Repository: git://github.com/xmldom/xmldom.git + +--- LICENSE --- +Copyright 2019 - present Christopher J. Brody and other contributors, as listed in: https://github.com/xmldom/xmldom/graphs/contributors +Copyright 2012 - 2017 @jindw and other contributors, as listed in: https://github.com/jindw/xmldom/graphs/contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + Package: @xterm/addon-unicode11@0.9.0 Declared license: MIT Selected license: MIT diff --git a/packages/cli/src/pi-transcript-format.ts b/packages/cli/src/pi-transcript-format.ts index 420b6a55b86..abd963f7cfb 100644 --- a/packages/cli/src/pi-transcript-format.ts +++ b/packages/cli/src/pi-transcript-format.ts @@ -48,6 +48,8 @@ export function fitLine(line: string, width: number): string { export function formatToolResultContent(content: ToolResultContent): string { switch (content.kind) { + case 'content': + return content.parts.map(formatToolResultContent).join('\n\n'); case 'text': return content.text; case 'json': diff --git a/packages/core/package.json b/packages/core/package.json index fdcb160935b..fd59dfef89f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -171,7 +171,9 @@ "./maka-wordmark": "./dist/maka-wordmark.js", "./test-only/async-primitives": "./dist/test-only/async-primitives.js", "./workhub-action-result": "./dist/workhub-action-result.js", - "./workhub-routing": "./dist/workhub-routing.js" + "./workhub-routing": "./dist/workhub-routing.js", + "./session-media": "./dist/session-media.js", + "./svg-preview": "./dist/svg-preview.js" }, "scripts": { "clean": "node ../../scripts/clean-paths.mjs dist tsconfig.tsbuildinfo src/model-metadata.generated.ts", @@ -182,6 +184,7 @@ "test:dist": "node --test \"dist/**/*.test.js\"" }, "dependencies": { + "@xmldom/xmldom": "^0.8.15", "intl-messageformat": "11.2.15", "marked": "18.0.11" } diff --git a/packages/core/src/__tests__/session-media.test.ts b/packages/core/src/__tests__/session-media.test.ts new file mode 100644 index 00000000000..be0b48571a8 --- /dev/null +++ b/packages/core/src/__tests__/session-media.test.ts @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + isSessionMediaInput, + isSessionContentPart, + sessionContentMarkdown, + SESSION_MEDIA_MAX_BASE64, +} from '../session-media.js'; +import { normalizeArtifactImagePreviewMime, normalizeMediaPreviewMime } from '../artifacts.js'; +import { prepareSvgPreview, SVG_PREVIEW_MAX_CHARS } from '../svg-preview.js'; + +test('media transport bounds binary input and only publishes durable image references', () => { + assert.ok(isSessionMediaInput({ kind: 'inline', mimeType: 'image/png', base64: 'AAAA' })); + for (const base64 of ['not base64', 'AA', 'AAAA=', 'A'.repeat(SESSION_MEDIA_MAX_BASE64 + 4)]) + assert.equal(isSessionMediaInput({ kind: 'inline', mimeType: 'image/png', base64 }), false); + assert.equal( + isSessionMediaInput({ kind: 'source', source: '/tmp/image.svg', extra: true }), + false, + ); + assert.equal(isSessionMediaInput({ kind: 'source', source: '/tmp/\0image.svg' }), false); + const part = { + kind: 'image' as const, + mimeType: 'image/svg+xml', + ref: { kind: 'session_file' as const, sessionId: 'session', relativePath: 'image-1' }, + }; + assert.ok(isSessionContentPart(part)); + assert.equal(sessionContentMarkdown(part), '\n\n![](maka://runtime/attachments/image-1)\n\n'); + assert.equal( + isSessionContentPart({ ...part, ref: { ...part.ref, relativePath: '../other' } }), + false, + ); +}); + +test('SVG display support does not expand model image admission', () => { + assert.equal(normalizeMediaPreviewMime('image/svg+xml'), 'image/svg+xml'); + assert.equal(normalizeMediaPreviewMime(undefined, 'drawing.svg'), 'image/svg+xml'); + assert.equal(normalizeArtifactImagePreviewMime('image/svg+xml'), null); +}); + +test('static SVG preserves geometry, text, gradients and safe inline styles', () => { + const source = + '你好 & Maka'; + const result = prepareSvgPreview(source); + assert.match(result, /linearGradient/); + assert.match(result, /url\(#paint\)/); + assert.match(result, /你好 & Maka/); + assert.match(result, /font-size:20px;fill:white/); + assert.match(prepareSvgPreview(''), /circle/); +}); + +test('SVG strips executable content, external dependencies and CSS escapes', () => { + const result = prepareSvgPreview( + '
html
', + ); + assert.doesNotMatch( + result, + /script|onload|foreignObject|https:| { + for (const source of [ + '', + ']>&x;', + '', + '', + '' + ''.repeat(66) + ''.repeat(66) + '', + '' + ''.repeat(4001) + '', + '' + ' '.repeat(SVG_PREVIEW_MAX_CHARS) + '', + ]) + assert.throws(() => prepareSvgPreview(source)); +}); + +test('ordered media survives canonical tool-result validation without allowing nested payloads', async () => { + const { decodeCanonicalToolResultContent } = await import('../tool-result-record-schema.js'); + const content = { + kind: 'content', + parts: [ + { kind: 'text', text: 'before' }, + { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session', relativePath: 'image-1' }, + }, + ], + }; + assert.deepEqual(decodeCanonicalToolResultContent(content), content); + assert.throws(() => decodeCanonicalToolResultContent({ kind: 'content', parts: [content] })); + assert.throws(() => + decodeCanonicalToolResultContent({ + kind: 'content', + parts: Array.from({ length: 65 }, () => ({ kind: 'text', text: 'a' })), + }), + ); +}); + +test('static SVG class styles preserve generated diagram colors without external CSS', () => { + const result = prepareSvgPreview( + '', + ); + assert.match(result, /class="box"/); + assert.match(result, /\.box\{fill:#4355d9;stroke:white\}/); + assert.doesNotMatch(result, /https:|@import|background/); +}); diff --git a/packages/core/src/artifacts.ts b/packages/core/src/artifacts.ts index 1cd87adc5c0..d320ca5aea8 100644 --- a/packages/core/src/artifacts.ts +++ b/packages/core/src/artifacts.ts @@ -66,6 +66,19 @@ export function normalizeArtifactImagePreviewMime( return ARTIFACT_IMAGE_PREVIEW_MIME_BY_EXTENSION[name.slice(dot).toLowerCase()] ?? null; } +/** Display-only formats. Model input admission remains raster-only. */ +export function normalizeMediaPreviewMime( + mimeType: string | undefined, + name?: string, +): string | null { + if ( + mimeType?.trim().toLowerCase() === 'image/svg+xml' || + (!mimeType?.trim() && name?.toLowerCase().endsWith('.svg')) + ) + return 'image/svg+xml'; + return normalizeArtifactImagePreviewMime(mimeType, name); +} + /** One metadata policy shared by preview admission and renderer presentation. */ export function resolveArtifactImagePreview( input: ArtifactImagePreviewInput, @@ -77,11 +90,11 @@ export function resolveArtifactImagePreview( return { kind: 'unsupported', reason: 'oversize' }; } if (input.mimeType) { - return normalizeArtifactImagePreviewMime(input.mimeType) + return normalizeMediaPreviewMime(input.mimeType) ? { kind: 'image', reason: 'mime_match' } : { kind: 'unsupported', reason: 'mime_disallowed' }; } - return normalizeArtifactImagePreviewMime(undefined, input.name) + return normalizeMediaPreviewMime(undefined, input.name) ? { kind: 'image', reason: 'ext_fallback' } : { kind: 'unsupported', reason: 'no_mime_no_ext' }; } diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index eec5e787621..4acb7f644ce 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -946,6 +946,7 @@ type ShellRunToolResultContent = )); export type ToolResultContent = + | { kind: 'content'; parts: import('./session-media.js').SessionContentPart[] } | { kind: 'text'; text: string; diff --git a/packages/core/src/session-media.ts b/packages/core/src/session-media.ts new file mode 100644 index 00000000000..fa2497e67ca --- /dev/null +++ b/packages/core/src/session-media.ts @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { StorageRef } from './events.js'; +import { isCanonicalArtifactEntityId } from './artifacts.js'; +import { formatAttachmentResourceRef } from './attachments.js'; + +/** Transport-neutral input. Only the Host may turn a source into a saved reference. */ +export type SessionMediaInput = + | { kind: 'inline'; mimeType: string; base64: string } + | { kind: 'source'; source: string }; + +export type SessionContentPart = + | { kind: 'text'; text: string } + | { kind: 'image'; mimeType: string; ref: Extract } + | { kind: 'file_diff'; paths: string[]; diff: string }; + +export const SESSION_CONTENT_MAX_PARTS = 64; +export const SESSION_MEDIA_MAX_BYTES = 2 * 1024 * 1024; +export const SESSION_MEDIA_MAX_BASE64 = 4 * Math.ceil(SESSION_MEDIA_MAX_BYTES / 3); + +export function isSessionMediaInput(value: unknown): value is SessionMediaInput { + if (!value || typeof value !== 'object') return false; + const v = value as Record; + if (v.kind === 'source') + return ( + Object.keys(v).length === 2 && + typeof v.source === 'string' && + v.source.length > 0 && + v.source.length <= 4096 && + !/[\x00-\x1f\x7f]/.test(v.source) + ); + return ( + v.kind === 'inline' && + Object.keys(v).length === 3 && + typeof v.mimeType === 'string' && + /^image\/[a-z0-9.+-]{1,64}$/.test(v.mimeType) && + typeof v.base64 === 'string' && + v.base64.length > 0 && + v.base64.length <= SESSION_MEDIA_MAX_BASE64 && + v.base64.length % 4 === 0 && + /^[A-Za-z0-9+/]*={0,2}$/.test(v.base64) + ); +} + +export function isSessionContentPart(value: unknown): value is SessionContentPart { + if (!value || typeof value !== 'object') return false; + const v = value as Record; + if (v.kind === 'text') + return Object.keys(v).length === 2 && typeof v.text === 'string' && v.text.length <= 8192; + if (v.kind === 'file_diff') + return ( + Object.keys(v).length === 3 && + Array.isArray(v.paths) && + v.paths.length > 0 && + v.paths.length <= 64 && + v.paths.every((p) => typeof p === 'string' && p.length <= 8192) && + typeof v.diff === 'string' && + v.diff.length <= 1024 * 1024 + ); + if ( + v.kind !== 'image' || + Object.keys(v).length !== 3 || + typeof v.mimeType !== 'string' || + !/^image\/[a-z0-9.+-]{1,64}$/.test(v.mimeType) || + !v.ref || + typeof v.ref !== 'object' + ) + return false; + const ref = v.ref as Record; + return ( + Object.keys(ref).length === 3 && + ref.kind === 'session_file' && + typeof ref.sessionId === 'string' && + ref.sessionId.length > 0 && + ref.sessionId.length <= 512 && + isCanonicalArtifactEntityId(ref.relativePath) + ); +} + +/** Compatibility projection: ordered media uses the existing durable Markdown transcript. */ +export function sessionContentMarkdown( + part: Exclude, +): string { + if (part.kind === 'text') return part.text; + const source = formatAttachmentResourceRef(part.ref); + return source ? `\n\n![](${source})\n\n` : ''; +} diff --git a/packages/core/src/svg-preview.ts b/packages/core/src/svg-preview.ts new file mode 100644 index 00000000000..73a8054053b --- /dev/null +++ b/packages/core/src/svg-preview.ts @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { DOMParser, XMLSerializer } from '@xmldom/xmldom'; + +export const SVG_PREVIEW_MAX_CHARS = 256 * 1024; +export const SVG_PREVIEW_MAX_NODES = 4000; +const NS = 'http://www.w3.org/2000/svg'; +const TAGS = new Set( + 'svg g defs symbol style title desc path rect circle ellipse line polyline polygon text tspan textPath linearGradient radialGradient stop clipPath mask pattern marker'.split( + ' ', + ), +); +const ATTRS = new Set( + 'id class viewBox width height x y x1 x2 y1 y2 cx cy r rx ry d points transform fill fill-opacity fill-rule stroke stroke-width stroke-opacity stroke-linecap stroke-linejoin stroke-dasharray stroke-dashoffset stroke-miterlimit opacity color font-family font-size font-weight font-style text-anchor dominant-baseline alignment-baseline dx dy rotate textLength lengthAdjust letter-spacing word-spacing gradientUnits gradientTransform spreadMethod offset stop-color stop-opacity fx fy fr clip-path clip-rule mask maskUnits maskContentUnits patternUnits patternContentUnits patternTransform marker-start marker-mid marker-end markerWidth markerHeight refX refY orient markerUnits preserveAspectRatio vector-effect paint-order visibility display'.split( + ' ', + ), +); +const STYLE = new Set( + 'fill fill-opacity fill-rule stroke stroke-width stroke-opacity stroke-linecap stroke-linejoin stroke-dasharray stroke-dashoffset opacity color font-family font-size font-weight font-style text-anchor dominant-baseline letter-spacing word-spacing stop-color stop-opacity'.split( + ' ', + ), +); + +/** Cheap byte sniff keeps ordinary unsupported binaries out of the XML parser. */ +export function isSvgPreviewCandidate(bytes: Uint8Array): boolean { + if (bytes.byteLength > SVG_PREVIEW_MAX_CHARS * 4) return false; + let offset = bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf ? 3 : 0; + while (offset < bytes.length) { + const byte = bytes[offset]; + if (byte !== 9 && byte !== 10 && byte !== 13 && byte !== 32) break; + offset++; + } + return bytes[offset] === 60; +} + +function safeValue(value: string): boolean { + // Reject CSS escapes, external URLs and expression syntax; fragment paint servers are supported. + return ( + !/[\\<>@]/.test(value) && + !/expression|javascript|data:|https?:|\/\*/i.test(value) && + !/url\s*\(/i.test(value.replace(/url\(\s*['"]?#[a-zA-Z_][\w:.-]*['"]?\s*\)/g, '')) + ); +} + +function staticDeclarations(source: string): string { + return source + .split(';') + .flatMap((declaration) => { + const colon = declaration.indexOf(':'); + const property = declaration.slice(0, colon).trim(); + const value = declaration.slice(colon + 1).trim(); + return colon > 0 && STYLE.has(property) && safeValue(value) ? [`${property}:${value}`] : []; + }) + .join(';'); +} + +/** A deliberately small static CSS subset, scoped by the SVG image document. */ +function staticStylesheet(source: string): string { + if (source.length > 32 * 1024) return ''; + const css = source.replace(/\/\*[\s\S]*?\*\//g, ''); + const rules: string[] = []; + for (const match of css.matchAll(/([^{}]+)\{([^{}]*)\}/g)) { + const selector = match[1]!.trim(); + if (!/^[a-zA-Z_.*#][\w.*# ,>:+-]*$/.test(selector)) continue; + const declarations = staticDeclarations(match[2]!); + if (declarations) rules.push(`${selector}{${declarations}}`); + } + return rules.join(''); +} + +/** Static SVG only. Never insert the returned document into the application's DOM. */ +export function prepareSvgPreview(source: string): string { + if (!source || source.length > SVG_PREVIEW_MAX_CHARS || / { + invalid = true; + }, + error: () => { + invalid = true; + }, + fatalError: () => { + invalid = true; + }, + }, + }).parseFromString(source, 'image/svg+xml'); + const root = document.documentElement; + if ( + invalid || + !root || + root.localName !== 'svg' || + (root.namespaceURI && root.namespaceURI !== NS) + ) + throw new Error('Invalid SVG document'); + // Iterate rather than recurse: pathological nesting cannot exhaust the call stack. + const pending: { node: Element; depth: number }[] = [{ node: root, depth: 0 }]; + let count = 0; + while (pending.length) { + const { node, depth } = pending.pop()!; + if (++count > SVG_PREVIEW_MAX_NODES || depth > 64) + throw new Error('SVG complexity exceeds preview limits'); + for (let i = node.attributes.length - 1; i >= 0; i--) { + const attribute = node.attributes.item(i)!; + const name = attribute.name; + if (name === 'style') { + const declarations = staticDeclarations(attribute.value); + if (declarations) node.setAttribute('style', declarations); + else node.removeAttribute(name); + } else if (!ATTRS.has(name) || !safeValue(attribute.value)) node.removeAttribute(name); + } + for (let child = node.firstChild; child; ) { + const next = child.nextSibling; + if (child.nodeType === 1) { + const element = child as Element; + if (!TAGS.has(element.localName) || (element.namespaceURI && element.namespaceURI !== NS)) + node.removeChild(child); + else if (element.localName === 'style') { + const css = staticStylesheet(element.textContent ?? ''); + if (!css) node.removeChild(child); + else { + element.textContent = css; + pending.push({ node: element, depth: depth + 1 }); + } + } else pending.push({ node: element, depth: depth + 1 }); + } else if (child.nodeType !== 3 && child.nodeType !== 4) node.removeChild(child); + child = next; + } + } + root.setAttribute('xmlns', NS); + const viewBox = root.hasAttribute('viewBox') + ? root.getAttribute('viewBox')!.trim().split(/[ ,]+/).map(Number) + : undefined; + if ( + viewBox && + (viewBox.length !== 4 || + viewBox.some((v) => !Number.isFinite(v)) || + viewBox[2]! <= 0 || + viewBox[3]! <= 0) + ) + throw new Error('Invalid SVG viewBox'); + for (const name of ['width', 'height']) { + const value = root.getAttribute(name); + if ( + value && + (!/^(?:\d+(?:\.\d*)?|\.\d+)(?:px|%)?$/.test(value) || + parseFloat(value) > 16384 || + parseFloat(value) <= 0) + ) + root.removeAttribute(name); + } + const width = parseFloat(root.getAttribute('width') || '300'); + const height = parseFloat(root.getAttribute('height') || '150'); + if (width * height > 32 * 1024 * 1024) throw new Error('SVG dimensions exceed preview limits'); + return new XMLSerializer().serializeToString(root); +} diff --git a/packages/core/src/tool-result-record-schema.ts b/packages/core/src/tool-result-record-schema.ts index 6f46fecc16c..7536ee09d46 100644 --- a/packages/core/src/tool-result-record-schema.ts +++ b/packages/core/src/tool-result-record-schema.ts @@ -17,6 +17,7 @@ * under the License. */ +import { isSessionContentPart, SESSION_CONTENT_MAX_PARTS } from './session-media.js'; import { decodeCanonicalShellToolResultContent, isSandboxDenialSignal, @@ -260,6 +261,14 @@ function isNonShellToolResultContent(value: unknown): value is ToolResultContent value.reason === 'stale_tool_result_pruned_before_compact' || value.reason === 'active_current_turn_tool_result_pruned_before_next_step') ); + case 'content': + return ( + Object.keys(value).length === 2 && + Array.isArray(value.parts) && + value.parts.length > 0 && + value.parts.length <= SESSION_CONTENT_MAX_PARTS && + value.parts.every(isSessionContentPart) + ); case 'image': return ( hasExactShape(value, IMAGE_SHAPE) && diff --git a/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts index f7b7c325dab..663bec04bd8 100644 --- a/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts +++ b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts @@ -123,6 +123,7 @@ async function fixture( return authority.store.create(input); }, findImageDelivery: authority.store.findImageDelivery, + getInSession: authority.store.getInSession, setImageDeliveryAttempt: authority.store.setImageDeliveryAttempt, }, admission: new SessionAdmissionGate(), @@ -614,7 +615,7 @@ test('copying a conversation carries delivery provenance and saved bytes; it doe } }); test('source readers reject unsafe MIME and redirects to private networks', async (t) => { - assert.throws(() => checkedChatImage(Buffer.from('')), /unsupported_mime/); + assert.throws(() => checkedChatImage(Buffer.from(' { res.writeHead(302, { location: 'http://169.254.169.254/latest/meta-data/' }); res.end(); @@ -933,3 +934,77 @@ test('a queued retry reports pending instead of replaying the previous failure', await f.close(); } }); + +test('executor inline media and Markdown use the same archive, deduplicate and replay after source deletion', async () => { + let writes = 0; + const f = await fixture(undefined, undefined, () => { + writes++; + }); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + try { + const input = { + kind: 'inline' as const, + mimeType: 'image/svg+xml', + base64: PNG.toString('base64'), + }; + const [a, b] = await Promise.all([ + f.service.publish(owner, input), + f.service.publish(owner, input), + ]); + assert.deepEqual(a, b); + assert.equal(a.kind, 'image'); + assert.equal(writes, 1); + if (a.kind !== 'image') throw new Error('Image not published'); + assert.equal(a.mimeType, 'image/png'); // Bytes, not the external MIME claim, are authoritative. + const read = await f.authority.store.readBinaryInSession(owner.sessionId, a.ref.relativePath); + assert.ok(read.ok); + if (read.ok) assert.equal(read.base64, PNG.toString('base64')); + const file = join(f.root, 'drawing.svg'); + await writeFile( + file, + '\n', + ); + const svg = await f.service.publish(owner, { kind: 'source', source: file }); + assert.equal(svg.kind, 'image'); + if (svg.kind !== 'image') throw new Error('SVG not published'); + const svgRead = await f.authority.store.readBinaryInSession( + owner.sessionId, + svg.ref.relativePath, + ); + assert.ok(svgRead.ok); + if (svgRead.ok) assert.equal(svgRead.mimeType, 'image/svg+xml'); + await rm(file); + assert.deepEqual(await f.service.publish(owner, { kind: 'source', source: file }), svg); + assert.equal(f.reads, 1); + assert.equal(f.leases, 0); + } finally { + await f.close(); + } +}); + +test('executor publication keeps invalid bytes, quota and network failures out of image references', async () => { + const f = await fixture({ sessionBytes: 10, workspaceBytes: 10 }); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + try { + const invalid = await f.service.publish(owner, { + kind: 'inline', + mimeType: 'image/png', + base64: Buffer.from('not an image').toString('base64'), + }); + assert.equal(invalid.kind, 'text'); + const quota = await f.service.publish(owner, { + kind: 'inline', + mimeType: 'image/png', + base64: PNG.toString('base64'), + }); + assert.deepEqual(quota, { kind: 'text', text: '[Media unavailable: quota_exceeded]' }); + f.allowRemote(false); + assert.deepEqual( + await f.service.publish(owner, { kind: 'source', source: 'https://example.com/image.svg' }), + { kind: 'text', text: '[Media unavailable: not_allowed]' }, + ); + assert.equal(f.leases, 0); + } finally { + await f.close(); + } +}); diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index e25dcc76501..956b0a81e53 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -108,7 +108,9 @@ export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. // Re-check against main before merging; sibling PRs may use the same next epoch. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 227 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 228 as const; +// 228: Tool results can contain ordered text, saved image and diff content parts. +// Older peers reject the new content variant in strict transcript decoders. // 227: Goal projections carry eventWait and support goal.event-wait.cancel. // Older peers cannot decode the wait summary or cancel a specific wait. // 226: assistant.allocate-workspace allocates a Host-owned fallback for projectless assistants. diff --git a/packages/runtime-host/src/server/chat-image-delivery.ts b/packages/runtime-host/src/server/chat-image-delivery.ts index d8af4f4fb9f..ec334f462df 100644 --- a/packages/runtime-host/src/server/chat-image-delivery.ts +++ b/packages/runtime-host/src/server/chat-image-delivery.ts @@ -17,6 +17,12 @@ * under the License. */ +import { + isSessionMediaInput, + type SessionMediaInput, + type SessionContentPart, +} from '@maka/core/session-media'; +import { checkedChatImage } from './chat-image-source.js'; import { createHash } from 'node:crypto'; import { ImageFileReadError } from '@maka/runtime/image-file-reader'; import type { TextCompleteEvent } from '@maka/core/events'; @@ -54,7 +60,7 @@ interface DeliveryJob { export interface ChatImageDeliveryPorts { readonly artifacts: Pick< InteractiveArtifactStoreWriter, - 'create' | 'findImageDelivery' | 'setImageDeliveryAttempt' + 'create' | 'findImageDelivery' | 'setImageDeliveryAttempt' | 'getInSession' >; readonly admission: SessionAdmissionGate; isPresent(sessionId: string): Promise; @@ -75,9 +81,70 @@ export class ChatImageDeliveryService { readonly #queue: DeliveryJob[] = []; readonly #abort = new AbortController(); #running = 0; + #inlineBytes = 0; #closed = false; constructor(private readonly ports: ChatImageDeliveryPorts) {} + /** Executor publication is Host-scoped; it never grants arbitrary client reads. */ + async publish( + owner: { sessionId: string; turnId: string; messageId: string }, + input: SessionMediaInput, + ): Promise { + if (!isSessionMediaInput(input)) + return { kind: 'text', text: '[Media unavailable: invalid input]' }; + const source = + input.kind === 'source' + ? input.source + : `inline:base64-sha256:${createHash('sha256').update(input.base64, 'ascii').digest('hex')}`; + const identity = { ...owner, source, loadRemote: true }; + const existing = await this.ports.artifacts.findImageDelivery( + owner.sessionId, + owner.turnId, + owner.messageId, + source, + ); + if (existing?.status !== 'ready') { + // Reserve queue capacity before decoding, including concurrent publishers waiting on metadata. + if ( + !this.#jobs.has(deliveryKey(identity)) && + input.kind === 'inline' && + this.#inlineBytes + (input.base64.length * 3) / 4 > 8 * 1024 * 1024 + ) + return { kind: 'text', text: '[Media unavailable: queue full]' }; + const bytes = + !this.#jobs.has(deliveryKey(identity)) && input.kind === 'inline' + ? Buffer.from(input.base64, 'base64') + : undefined; + if ( + bytes && + bytes.toString('base64') !== + (input as Extract).base64 + ) + return { kind: 'text', text: '[Media unavailable: invalid encoding]' }; + if (!this.#enqueue(identity, bytes)) + return { kind: 'text', text: '[Media unavailable: queue full]' }; + await this.#jobs.get(deliveryKey(identity))?.done; + } + const saved = await this.ports.artifacts.findImageDelivery( + owner.sessionId, + owner.turnId, + owner.messageId, + source, + ); + if (saved?.status !== 'ready') + return { + kind: 'text', + text: `[Media unavailable: ${saved?.status === 'failed' ? saved.reason : 'not saved'}]`, + }; + const { record } = await this.ports.artifacts.getInSession(owner.sessionId, saved.artifactId); + if (!record?.mimeType) return { kind: 'text', text: '[Media unavailable: missing artifact]' }; + return { + kind: 'image', + mimeType: record.mimeType, + ref: { kind: 'session_file', sessionId: owner.sessionId, relativePath: saved.artifactId }, + }; + } + observe(sessionId: string, event: TextCompleteEvent): void { if (this.#closed || event.text.length > IMAGE_MARKDOWN_MAX_LENGTH) return; // Reference destinations can grow during streaming. Only settled text @@ -121,10 +188,16 @@ export class ChatImageDeliveryService { : { status: 'failed', reason: 'queue_full' }; } - #enqueue(identity: DeliveryIdentity): boolean { + #enqueue(identity: DeliveryIdentity, inline?: Uint8Array): boolean { const key = deliveryKey(identity); if (this.#jobs.has(key)) return true; - if (this.#closed || this.#jobs.size >= 128) return false; + if ( + this.#closed || + this.#jobs.size >= 128 || + this.#inlineBytes + (inline?.byteLength ?? 0) > 8 * 1024 * 1024 + ) + return false; + this.#inlineBytes += inline?.byteLength ?? 0; let settle!: () => void; const done = new Promise((resolve) => { settle = resolve; @@ -132,6 +205,7 @@ export class ChatImageDeliveryService { const residency = this.ports.acquireResidency(); const finish = () => { this.#jobs.delete(key); + this.#inlineBytes -= inline?.byteLength ?? 0; residency.release(); settle(); }; @@ -140,7 +214,7 @@ export class ChatImageDeliveryService { cancel: finish, run: async () => { try { - await this.#capture(identity); + await this.#capture(identity, inline); } catch (error) { if (!this.#abort.signal.aborted) { if (error instanceof ImagePersistenceError) this.ports.persistenceFailed(error.cause); @@ -166,7 +240,7 @@ export class ChatImageDeliveryService { }); } } - async #capture(identity: DeliveryIdentity): Promise { + async #capture(identity: DeliveryIdentity, inline?: Uint8Array): Promise { const { sessionId, turnId, messageId, source } = identity; const existing = await this.ports.artifacts.findImageDelivery( sessionId, @@ -199,8 +273,10 @@ export class ChatImageDeliveryService { const signal = AbortSignal.any([this.#abort.signal, readAbort.signal]); let image: ChatImageBytes; try { - const path = localImagePath(source); - if (path !== undefined) { + const path = inline ? undefined : localImagePath(source); + if (inline) { + image = checkedChatImage(inline); + } else if (path !== undefined) { image = await abortable(() => this.ports.readLocalImage(sessionId, path, signal), signal); } else { if (!identity.loadRemote || !(await this.ports.canLoadRemote(sessionId))) diff --git a/packages/runtime-host/src/server/chat-image-source.ts b/packages/runtime-host/src/server/chat-image-source.ts index 34596f90732..451a12fcec8 100644 --- a/packages/runtime-host/src/server/chat-image-source.ts +++ b/packages/runtime-host/src/server/chat-image-source.ts @@ -77,7 +77,7 @@ export async function downloadChatImage( const response = await fetch(url, { targetPolicy: 'public', signal, - headers: { accept: 'image/png,image/jpeg,image/webp,image/gif' }, + headers: { accept: 'image/png,image/jpeg,image/webp,image/gif,image/svg+xml' }, redirect: 'manual', credentials: 'omit', }); diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index fc1c3df3d69..525b94341db 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -1466,6 +1466,7 @@ export async function createExecutionRuntimeHostComposition( : {}), ...(factoryContext.systemPrompt ? { instructions: factoryContext.systemPrompt } : {}), binding, + publishMedia: (owner, media) => imageDelivery!.publish(owner, media), }), }; }, diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index a365090b7df..c5c8dd29e84 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -4210,3 +4210,57 @@ function invocationOpenedEvent(run: SeededRun = {}): RuntimeEvent { opening: invocationOpening(run), }); } + +test('conversation copies collect and rewrite every image in ordered media content', () => { + const message: StoredMessage = { + type: 'tool_result', + id: 'result-media', + turnId: 'turn-1', + ts: 1, + toolUseId: 'tool-media', + isError: false, + content: { + kind: 'content', + parts: [ + { kind: 'text', text: 'before ![](maka://runtime/attachments/svg-original)' }, + { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session-source', relativePath: 'svg-original' }, + }, + { kind: 'text', text: 'after' }, + ], + }, + }; + assert.deepEqual( + [ + ...collectConversationCopySessionFileRefs({ + sourceSessionId: 'session-source', + messages: [message], + runtimeEvents: [], + }), + ], + ['svg-original'], + ); + const copied = rewriteConversationCopyMessage(message, { + sourceSessionId: 'session-source', + targetSessionId: 'session-target', + artifactIds: new Map([['svg-original', 'svg-copy']]), + relativePaths: new Map(), + runtimeEventIds: new Map(), + runIds: new Map(), + providerTraceIds: new Map(), + }); + assert.equal(copied.type, 'tool_result'); + if (copied.type !== 'tool_result' || copied.content.kind !== 'content') + throw new Error('Expected media'); + assert.deepEqual(copied.content.parts[1], { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session-target', relativePath: 'svg-copy' }, + }); + assert.deepEqual(copied.content.parts[0], { + kind: 'text', + text: 'before ![](maka://runtime/attachments/svg-copy)', + }); +}); diff --git a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts index 822b4b6c96b..64836f84a23 100644 --- a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts +++ b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts @@ -649,3 +649,55 @@ async function collect(events: AsyncIterable): Promise { + const root = new Context(); + const service = new PluginExecutorService(root); + const publications: unknown[] = []; + root + .extend({ + maka: { rootId: 'profile', packageId: 'fixture', entryId: 'provider', generation: 1 }, + }) + .executors.register({ + id: 'media', + capabilities: { toolActivity: true }, + execute: async (_request, context) => { + context.emit({ type: 'tool_start', toolCallId: 'draw', name: 'draw' }); + const image = await context.publishMedia!({ + kind: 'inline', + mimeType: 'image/svg+xml', + base64: 'PHN2Zy8+', + }); + context.emit({ + type: 'tool_result', + toolCallId: 'draw', + content: { kind: 'content', parts: [{ kind: 'text', text: 'drawing' }, image] }, + }); + return { status: 'completed', text: 'done' }; + }, + }); + const backend = new PluginExecutorBackend({ + sessionId: 'session-a', + cwd: '/workspace', + binding: service.bind('session-a', 'media'), + publishMedia: async (owner, input) => { + publications.push({ owner, input }); + return { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: owner.sessionId, relativePath: 'saved-svg' }, + }; + }, + }); + try { + const events = await collect(backend.send({ turnId: 'turn-a', text: 'draw' })); + assert.equal(publications.length, 1); + const result = events.find((event) => event.type === 'tool_result'); + assert.equal(result?.content.kind, 'content'); + assert.doesNotMatch(JSON.stringify(events), /PHN2Zy8\+/); + assert.match(JSON.stringify(events), /saved-svg/); + } finally { + await backend.dispose(); + await root.fiber.dispose(); + } +}); diff --git a/packages/runtime/src/conversation-copy.ts b/packages/runtime/src/conversation-copy.ts index ce29a964e1c..2c8a23f60b1 100644 --- a/packages/runtime/src/conversation-copy.ts +++ b/packages/runtime/src/conversation-copy.ts @@ -191,6 +191,7 @@ function collectConversationCopyStorageRefs( const refs: StorageRef[] = []; const addContent = (content: ToolResultContent): void => { if (content.kind === 'image') refs.push(content.ref); + if (content.kind === 'content') for (const part of content.parts) addContent(part); }; const addSerialized = (value: unknown): void => { if (isArchivedToolResultPlaceholder(value)) return; @@ -1892,6 +1893,23 @@ function rewriteToolResultContent( return content; } } + if (content.kind === 'content') { + return { + ...content, + parts: content.parts.map((part) => { + if (part.kind === 'text') + return { + ...part, + text: rewriteAttachmentResourceRefs(part.text, references.artifactIds), + }; + if (part.kind !== 'image') return part; + const ref = rewriteStorageRef(part.ref, references); + if (ref.kind !== 'session_file') + throw new Error('Copied media must retain a Session artifact reference'); + return { ...part, ref }; + }), + }; + } if (content.kind === 'image') { return { ...content, ref: rewriteStorageRef(content.ref, references) }; } diff --git a/packages/runtime/src/filesystem-worker/protocol.ts b/packages/runtime/src/filesystem-worker/protocol.ts index bf1512c4fe5..ccfa8314c35 100644 --- a/packages/runtime/src/filesystem-worker/protocol.ts +++ b/packages/runtime/src/filesystem-worker/protocol.ts @@ -179,7 +179,7 @@ export const FilesystemWorkerResultSchema = z.discriminatedUnion('kind', [ .object({ kind: z.literal('read_image'), base64: z.string(), - mimeType: z.enum(['image/png', 'image/jpeg', 'image/gif', 'image/webp']), + mimeType: z.enum(['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/svg+xml']), }) .strict(), z diff --git a/packages/runtime/src/image-file.ts b/packages/runtime/src/image-file.ts index c1f0ec5a71f..95c2ae23dc8 100644 --- a/packages/runtime/src/image-file.ts +++ b/packages/runtime/src/image-file.ts @@ -28,6 +28,7 @@ import { READ_IMAGE_TOO_LARGE_MESSAGE, sniffAttachmentMimeType, } from '@maka/core/attachments'; +import { prepareSvgPreview, isSvgPreviewCandidate } from '@maka/core/svg-preview'; import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; export interface WorkspaceFileReadOptions { @@ -36,7 +37,12 @@ export interface WorkspaceFileReadOptions { } const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp']); -export type ImageMimeType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp'; +export type ImageMimeType = + | 'image/png' + | 'image/jpeg' + | 'image/gif' + | 'image/webp' + | 'image/svg+xml'; export class ImageFileError extends Error { constructor( @@ -100,6 +106,14 @@ export function validateImageBytes( if (bytes.length > (purpose === 'chat' ? ARTIFACT_IMAGE_PREVIEW_MAX_BYTES : MAX_READ_IMAGE_BYTES)) throw imageTooLargeError(purpose); const mimeType = sniffImageMime(bytes); + if (!mimeType && purpose === 'chat' && isSvgPreviewCandidate(bytes)) { + try { + prepareSvgPreview(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + return { bytes, mimeType: 'image/svg+xml' }; + } catch { + /* Continue to the common unsupported-image failure. */ + } + } if (!mimeType) throw new ImageFileError( 'ERR_INVALID_IMAGE', diff --git a/packages/runtime/src/plugin-executor-backend.ts b/packages/runtime/src/plugin-executor-backend.ts index 16083e7080b..8e70386a546 100644 --- a/packages/runtime/src/plugin-executor-backend.ts +++ b/packages/runtime/src/plugin-executor-backend.ts @@ -61,6 +61,10 @@ export interface PluginExecutorBackendInput { readonly model?: string; readonly thinkingLevel?: ThinkingLevel; readonly binding: PluginExecutorBinding; + readonly publishMedia?: ( + owner: { sessionId: string; turnId: string; messageId: string }, + input: import('@maka/core/session-media').SessionMediaInput, + ) => Promise; readonly newId?: () => string; readonly now?: () => number; } @@ -75,6 +79,7 @@ export class PluginExecutorBackend implements AgentBackend { readonly #model?: string; readonly #thinkingLevel?: ThinkingLevel; readonly #binding: PluginExecutorBinding; + readonly #publishMedia: PluginExecutorBackendInput['publishMedia']; readonly #newId: () => string; readonly #now: () => number; readonly #active = new Set(); @@ -88,6 +93,7 @@ export class PluginExecutorBackend implements AgentBackend { this.#model = input.model; this.#thinkingLevel = input.thinkingLevel; this.#binding = input.binding; + this.#publishMedia = input.publishMedia; this.#newId = input.newId ?? randomUUID; this.#now = input.now ?? Date.now; } @@ -209,6 +215,12 @@ export class PluginExecutorBackend implements AgentBackend { }, { signal, + ...(this.#publishMedia + ? { + onPublishMedia: (media: import('@maka/core/session-media').SessionMediaInput) => + this.#publishMedia!({ sessionId: this.sessionId, turnId, messageId }, media), + } + : {}), onSettled: () => { settled = true; }, @@ -571,9 +583,11 @@ export class PluginExecutorBackend implements AgentBackend { providerExecuted: true, isError: event.isError ?? false, content: - event.content.kind === 'text' - ? event.content - : { kind: 'file_diff', paths: [...event.content.paths], diff: event.content.diff }, + event.content.kind === 'content' + ? { kind: 'content', parts: [...event.content.parts] } + : event.content.kind === 'text' + ? event.content + : { kind: 'file_diff', paths: [...event.content.paths], diff: event.content.diff }, }); } diff --git a/packages/runtime/src/plugin-executor-service.ts b/packages/runtime/src/plugin-executor-service.ts index 123196b410c..c011675b9c9 100644 --- a/packages/runtime/src/plugin-executor-service.ts +++ b/packages/runtime/src/plugin-executor-service.ts @@ -17,6 +17,13 @@ * under the License. */ +import { + isSessionMediaInput, + isSessionContentPart, + SESSION_CONTENT_MAX_PARTS, + type SessionMediaInput, + type SessionContentPart, +} from '@maka/core/session-media'; import { isExecutorConfiguration, normalizeCatalogEntry, @@ -105,6 +112,7 @@ export type PluginExecutorOutputEvent = /** Durable tool-result shapes that an external executor may publish directly. */ export type PluginExecutorToolResultContent = + | { readonly kind: 'content'; readonly parts: readonly SessionContentPart[] } | { readonly kind: 'text'; readonly text: string } | { readonly kind: 'file_diff'; readonly paths: readonly string[]; readonly diff: string }; @@ -147,6 +155,7 @@ export type PluginExecutorResult = export interface PluginExecutorContext { readonly signal: AbortSignal; emit(event: PluginExecutorOutputEvent): void; + publishMedia?(input: SessionMediaInput): Promise; /** Request one provider-defined permission choice through Maka's hosted form authority. */ requestPermission( request: PluginExecutorPermissionRequest, @@ -190,6 +199,7 @@ export interface PluginExecutorProvider { } export interface PluginExecutorExecutionOptions { + readonly onPublishMedia?: (input: SessionMediaInput) => Promise; readonly signal?: AbortSignal; /** Called when the provider returns a terminal result, before the Runtime validates it. */ readonly onSettled?: () => void; @@ -537,12 +547,33 @@ export class PluginExecutorService extends Service { emit: (event) => { if (entry.retired) return; const normalized = normalizeOutputEvent(event, entry.provider.capabilities); + if ( + normalized.type === 'tool_result' && + normalized.content.kind === 'content' && + normalized.content.parts.some( + (part) => + part.kind === 'image' && part.ref.sessionId !== normalizedRequest.sessionId, + ) + ) + throw new TypeError('Executor media is outside the current Session'); try { options.onEvent?.(normalized); } catch { // A presentation observer must not change external execution. } }, + publishMedia: async (input) => { + if (signal.aborted || entry.retired) + return { kind: 'text', text: '[Media unavailable: execution ended]' }; + if (!isSessionMediaInput(input)) + return { kind: 'text', text: '[Media unavailable: invalid input]' }; + const part = (await options.onPublishMedia?.(Object.freeze({ ...input }))) ?? { + kind: 'text' as const, + text: '[Media preview unavailable]', + }; + if (!isSessionContentPart(part)) throw new TypeError('Invalid published media'); + return part; + }, requestPermission: async (request) => { if (signal.aborted || entry.retired) return Object.freeze({ outcome: 'cancelled' }); const normalized = normalizePermissionRequest(request); @@ -766,13 +797,20 @@ function normalizeOutputEvent( type: event.type, toolCallId: event.toolCallId, content: - event.content.kind === 'text' - ? Object.freeze({ kind: 'text' as const, text: event.content.text }) - : Object.freeze({ - kind: 'file_diff' as const, - paths: Object.freeze([...event.content.paths]), - diff: event.content.diff, - }), + event.content.kind === 'content' + ? Object.freeze({ + kind: 'content' as const, + parts: Object.freeze( + event.content.parts.map((part) => Object.freeze(structuredClone(part))), + ), + }) + : event.content.kind === 'text' + ? Object.freeze({ kind: 'text' as const, text: event.content.text }) + : Object.freeze({ + kind: 'file_diff' as const, + paths: Object.freeze([...event.content.paths]), + diff: event.content.diff, + }), ...(event.isError === undefined ? {} : { isError: event.isError }), }); } @@ -781,6 +819,14 @@ function normalizeOutputEvent( function isPluginToolResultContent(value: PluginExecutorToolResultContent): boolean { if (!value || typeof value !== 'object') return false; + if (value.kind === 'content') + return ( + Array.isArray(value.parts) && + value.parts.length > 0 && + value.parts.length <= SESSION_CONTENT_MAX_PARTS && + value.parts.every(isSessionContentPart) && + Buffer.byteLength(JSON.stringify(value.parts), 'utf8') <= 1024 * 1024 + ); if (value.kind === 'text') return isSafeEventText(value.text); return ( value.kind === 'file_diff' && diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index 0ab11a1756c..d31244abf32 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -3469,6 +3469,7 @@ export function deriveToolResultStatus( case 'file_diff': case 'file_write': case 'image': + case 'content': case 'summary': case 'archived_tool_result': case 'web_search': diff --git a/packages/storage/src/artifact-store.ts b/packages/storage/src/artifact-store.ts index 745c94db5b6..43f8295c922 100644 --- a/packages/storage/src/artifact-store.ts +++ b/packages/storage/src/artifact-store.ts @@ -17,6 +17,7 @@ * under the License. */ +import { prepareSvgPreview, isSvgPreviewCandidate } from '@maka/core/svg-preview'; import { assertImageArchiveQuota, planImageArchive } from './artifact-image-storage.js'; import { DEFAULT_IMAGE_ARCHIVE_LIMITS, @@ -1415,16 +1416,17 @@ function isNotFound(error: unknown): boolean { } function sniffAllowedBinaryMime(bytes: Uint8Array): string | null { - // Core owns the binary signatures, shared with the attachment and image-read - // paths so the three cannot drift. SVG needs a wider text scan than a fixed - // prefix, so it stays local to this reader. const sniffed = sniffAttachmentMimeType(bytes); if (sniffed) return sniffed; - const leading = new TextDecoder('utf-8', { fatal: false }) - .decode(bytes.slice(0, Math.min(bytes.length, 512))) - .trimStart(); - if (/^]/i.test(leading) || /^<\?xml[\s\S]*]/i.test(leading)) - return 'image/svg+xml'; + // Share SVG validation with capture and display, including XML declarations/comments. + if (isSvgPreviewCandidate(bytes)) { + try { + prepareSvgPreview(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + return 'image/svg+xml'; + } catch { + /* Unknown or unsupported file content. */ + } + } return null; } diff --git a/packages/ui/scripts/benchmark-session-media.mjs b/packages/ui/scripts/benchmark-session-media.mjs new file mode 100644 index 00000000000..165cb9cc3df --- /dev/null +++ b/packages/ui/scripts/benchmark-session-media.mjs @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Chromium owns image requests and CSP enforcement; fake DOM tests cannot +// detect an allowed Markdown URL that the desktop policy blocks. +// Run after building @maka/ui: node --test packages/ui/scripts/markdown-images.browser.test.mjs +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import vm from 'node:vm'; +import { transformSync } from 'esbuild'; + +// A narrow allocation/CPU regression benchmark, not a browser frame-rate claim. +const root = fileURLToPath(new URL('../../../', import.meta.url)); +const path = 'packages/ui/src/markdown-body.tsx'; +const baseline = process.argv[2] ?? 'HEAD'; +const previous = execFileSync('git', ['show', `${baseline}:${path}`], { cwd: root, encoding: 'utf8' }); +const current = readFileSync(new URL('../src/markdown-body.tsx', import.meta.url), 'utf8'); +function compile(source) { + const start = source.indexOf('export const MAX_AUTOMATIC_MERMAID_DIAGRAMS'); + const end = source.indexOf('// Whether the prose', start); + if (start < 0 || end < 0) throw new Error('Diagram budget boundary not found'); + const { code } = transformSync(source.slice(start, end), { loader: 'ts', format: 'cjs', target: 'es2022' }); + const context = { module: { exports: {} }, exports: {} }; + vm.runInNewContext(code, context); + return context.module.exports.applyMermaidRenderBudget; +} +const before = compile(previous), after = compile(current); +const cases = { + prose: 'Some ordinary message text, with no diagram.\n'.repeat(150), + images: '![Screenshot](/tmp/result.png)\n\n'.repeat(30), + code: '```ts\nconst x = 1;\n```\n\n'.repeat(30), + mermaid: '```mermaid\ngraph TD; A-->B\n```\n\n'.repeat(5), +}; +function run(fn, source) { + const start = performance.now(); + for (let i = 0; i < 3000; i++) fn(source); + return performance.now() - start; +} +for (const [name, source] of Object.entries(cases)) { + for (let i = 0; i < 500; i++) { before(source); after(source); } + const first = [], second = []; + for (let i = 0; i < 7; i++) { + if (i % 2) { second.push(run(after, source)); first.push(run(before, source)); } + else { first.push(run(before, source)); second.push(run(after, source)); } + } + first.sort((a, b) => a - b); second.sort((a, b) => a - b); + console.log(JSON.stringify({ case: name, iterations: 3000, baseline, beforeMs: first[3], afterMs: second[3], ratio: second[3] / first[3] })); +} diff --git a/packages/ui/scripts/markdown-images.browser.test.mjs b/packages/ui/scripts/markdown-images.browser.test.mjs index 4f0d873537c..734004f904b 100644 --- a/packages/ui/scripts/markdown-images.browser.test.mjs +++ b/packages/ui/scripts/markdown-images.browser.test.mjs @@ -46,6 +46,11 @@ before(async () => { await new Promise(resolve => imageServer.listen(0, '127.0.0.1', resolve)); imageUrl = `http://127.0.0.1:${imageServer.address().port}`; const destinations = { + 'svg-saved': '![SVG](maka://runtime/attachments/svg-image)', + "svg-fence": "```svg\nMaka SVG\n```", + "svg-fence-offscreen": "```svg\nMaka SVG\n```", + "svg-fence-streaming": "```svg\nMaka SVG\n```", + 'links-formats': [ `![Screenshot](${imageUrl}/image.png)`, `![Title](${imageUrl}/title.png "Screenshot title")`, @@ -100,6 +105,7 @@ before(async () => { '![Screenshot](${imageUrl}/'+(mode==='retry' ? 'retry.png' : 'image.png')+')'); const root=createRoot(document.getElementById('root')); const readBytes=async(_session,artifactId)=>{ + if(mode==='svg-saved') { window.imageReads++; return {ok:true,mimeType:'image/svg+xml',base64:'PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNDAiIGhlaWdodD0iMTIwIiB2aWV3Qm94PSIwIDAgMjQwIDEyMCIgb25sb2FkPSJ3aW5kb3cuc3ZnRXhlY3V0ZWQ9dHJ1ZSI+PHNjcmlwdD53aW5kb3cuc3ZnRXhlY3V0ZWQ9dHJ1ZTwvc2NyaXB0PjxyZWN0IHdpZHRoPSIyNDAiIGhlaWdodD0iMTIwIiByeD0iMTIiIGZpbGw9IiM0MzU1ZDkiLz48dGV4dCB4PSIyMCIgeT0iNzAiIGZpbGw9IndoaXRlIiBmb250LXNpemU9IjI4Ij5NYWthIFNWRzwvdGV4dD48L3N2Zz4='}; } if(captured.has(artifactId)) return captured.get(artifactId); reads++; window.imageReads=reads; return (mode==='attachment' || mode==='read-retry-saved') && reads===1 ? {ok:false,reason:'read_failed'} : {ok:true,base64:mode.endsWith('corrupt-saved') && reads===1 ? 'iVBORw0KGgo=' : '${png.toString('base64')}',mimeType:'image/png'}; @@ -128,12 +134,12 @@ before(async () => { }; const markdown=(streaming)=>React.createElement(ImageMessageProvider,{identity:{turnId:'turn',messageId},streaming}, React.createElement(Markdown,{text,streaming,settledText:race ? text : undefined,density:'compact',imageDisplay:mode.startsWith('links-') ? 'link' : undefined})); - const render=(streaming=race || mode==='links-streaming')=>root.render( + const render=(streaming=race || mode==='links-streaming' || mode==='svg-fence-streaming')=>root.render( React.createElement(Theme,{theme:makaTheme,mode:'light'}, React.createElement(LocaleProvider,{locale:'en'}, React.createElement(SessionAttachmentProvider,{sessionId:'session',readBytes}, React.createElement(ImageDeliveryProvider,{sessionId:'session',resolve:resolveDelivery}, - React.createElement('div',{style:mode==='offscreen' ? {paddingTop:'2500px'} : {}}, + React.createElement('div',{style:(mode==='offscreen' || mode==='svg-fence-offscreen') ? {paddingTop:'2500px'} : {}}, markdown(streaming))))))); window.nextMessage=()=>{messageId='next-message'; render(false);}; window.finishStream=()=>render(false); @@ -165,7 +171,7 @@ before(async () => { }); await new Promise(resolve => appServer.listen(0, '127.0.0.1', resolve)); appUrl = `http://127.0.0.1:${appServer.address().port}`; - browser = await chromium.launch({ headless: true }); + browser = await chromium.launch({ headless: true, ...(process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH } : {}) }); }); after(async () => { @@ -423,3 +429,39 @@ for (const scenario of ['angle-saved', 'title-saved', 'escaped-saved', 'referenc } finally { await page.close(); } }); } + + +for (const mode of ['svg-saved', 'svg-fence']) { + test(`SVG ${mode} displays through static image context and strips executable content`, async () => { + const page = await pageFor(mode); + try { + await page.waitForFunction(() => { const img = document.querySelector('#root img'); return img?.complete && img.naturalWidth === 240; }); + assert.equal(await page.evaluate(() => window.svgExecuted), undefined); + const source = await page.locator('#root img').getAttribute('src'); + assert.doesNotMatch(decodeURIComponent(source), / { + const streaming = await pageFor('svg-fence-streaming'); + const offscreen = await pageFor('svg-fence-offscreen'); + try { + await streaming.locator('[data-maka-contract="markdown"]').waitFor(); + assert.equal(await streaming.locator('[data-kind="svg-preview"]').count(), 0); + await streaming.evaluate(() => window.finishStream()); + await streaming.locator('[data-kind="svg-preview"] img').waitFor(); + await offscreen.locator('[data-kind="svg-preview"]').waitFor({state:'attached'}); + assert.equal(await offscreen.locator('#root img').count(), 0); + await offscreen.evaluate(() => window.scrollTo(0, document.body.scrollHeight)); + await offscreen.locator('#root img').waitFor(); + } finally { await streaming.close(); await offscreen.close(); } +}); diff --git a/packages/ui/src/__tests__/attachment-image.test.tsx b/packages/ui/src/__tests__/attachment-image.test.tsx index 1203e69db4b..6b61496a6a1 100644 --- a/packages/ui/src/__tests__/attachment-image.test.tsx +++ b/packages/ui/src/__tests__/attachment-image.test.tsx @@ -271,3 +271,37 @@ test('renders an attachment when a streaming Markdown image becomes complete', a assert.ok(image); assert.equal(image.getAttribute('src'), 'data:image/png;base64,c3RyZWFt'); }); + + +test('text, tool media and virtualized remounts share a bounded saved-image cache', async () => { + const { SessionMediaImage } = await import('../session-media-image.js'); + const { ToolResultPreview } = await import('../tool-activity/tool-result-preview.js'); + const { container, root } = domRoot(); + let reads = 0; + const readBytes: ReadAttachmentBytes = async () => { + reads++; + return { ok: true, base64: 'AAAA', mimeType: 'image/png' }; + }; + const part = { kind: 'image' as const, mimeType: 'image/png', ref: { kind: 'session_file' as const, sessionId: 'session-1', relativePath: 'image-1' } }; + const render = (visible: boolean) => root.render( + {visible && <>} + ); + await act(async () => render(true)); + assert.equal(container.querySelectorAll('img').length, 2); + assert.equal(reads, 1); + await act(async () => render(false)); + await act(async () => render(true)); + assert.equal(reads, 1, 'remount must not reread or re-encode saved bytes'); +}); + +test('saved SVG previews sanitize content before image state and cache identical conversions', async () => { + const { mediaImageSource } = await import('../media-image-source.js'); + const svg = ''; + const result = { ok: true as const, mimeType: 'image/svg+xml', base64: Buffer.from(svg).toString('base64') }; + const [first, second] = await Promise.all([mediaImageSource(result), mediaImageSource(result)]); + assert.equal(first, second); + assert.ok(first); + assert.doesNotMatch(decodeURIComponent(first), /script|onload/); + assert.match(decodeURIComponent(first), /rect/); + await assert.rejects(mediaImageSource({ ...result, base64: Buffer.from('not SVG').toString('base64') })); +}); diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts index 86f35bb5b49..ec7d320c080 100644 --- a/packages/ui/src/__tests__/markdown-body.test.ts +++ b/packages/ui/src/__tests__/markdown-body.test.ts @@ -642,3 +642,18 @@ it('memory citations route internally and never navigate as external URLs', () = assert.equal((markup.match(/data-maka-uri-kind="memory-original"/g) ?? []).length, 2); assert.doesNotMatch(markup, /unsafe-scheme|href="memory-original/); }); + + +it('SVG auto-preview has a separate bounded budget without rewriting streaming source', () => { + const small = '```svg\n\n```'; + const budgeted = applyMermaidRenderBudget([small, small, small, small].join('\n\n')); + assert.equal((budgeted.match(/```svg\n/g) ?? []).length, 3); + assert.equal((budgeted.match(/```makasvgdeferred\n/g) ?? []).length, 1); + assert.match(applyMermaidRenderBudget('```svg\n' + ' '.repeat(65536) + '\n```'), /```makasvgdeferred/); +}); + + +it('diagram budgets do not rewrite SVG examples nested in another code fence', () => { + const example = '````markdown\n' + '```svg\n\n```\n'.repeat(5) + '````'; + assert.equal(applyMermaidRenderBudget(example), example); +}); diff --git a/packages/ui/src/artifact-preview-registry.ts b/packages/ui/src/artifact-preview-registry.ts index 89158dff5ca..37263f897d6 100644 --- a/packages/ui/src/artifact-preview-registry.ts +++ b/packages/ui/src/artifact-preview-registry.ts @@ -21,7 +21,7 @@ import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES, - normalizeArtifactImagePreviewMime, + normalizeMediaPreviewMime, resolveArtifactImagePreview, type ArtifactBinaryReadResult, type ArtifactImagePreviewInput, @@ -53,7 +53,7 @@ function decideImagePostLoad(input: { if (exceedsImagePayloadCap(input.base64)) { return { kind: 'unsupported', reason: 'oversize' }; } - const safeMime = normalizeArtifactImagePreviewMime(input.mimeType); + const safeMime = normalizeMediaPreviewMime(input.mimeType); if (!safeMime) { return { kind: 'unsupported', reason: 'mime_disallowed' }; } diff --git a/packages/ui/src/attachment-image.tsx b/packages/ui/src/attachment-image.tsx index dc52a3b3311..cb9f71a6d3a 100644 --- a/packages/ui/src/attachment-image.tsx +++ b/packages/ui/src/attachment-image.tsx @@ -27,7 +27,7 @@ import { type ReactNode, } from 'react'; import type { ArtifactBinaryReadResult } from '@maka/core/artifacts'; -import { decideImageReadOutcome } from './artifact-preview-registry.js'; +import { mediaImageSource } from './media-image-source.js'; /** Host capability for reading bytes from the Runtime Host attachment authority. */ export type ReadAttachmentBytes = ( @@ -37,7 +37,7 @@ export type ReadAttachmentBytes = ( type SessionAttachmentContextValue = { sessionId: string; - loadImage: (sessionId: string, artifactId: string) => Promise; + loadImage: (sessionId: string, artifactId: string, retry?: boolean) => Promise; }; const SessionAttachmentContext = createContext(undefined); @@ -53,18 +53,28 @@ export function SessionAttachmentProvider(props: { const readBytes = props.readBytes; if (!readBytes) return undefined; const pending = new Map>(); + const ready = new Map(); + let readyChars = 0; return { sessionId: props.sessionId, - loadImage(sessionId: string, artifactId: string) { + loadImage(sessionId: string, artifactId: string, retry = false) { const key = `${sessionId}\0${artifactId}`; + if (retry) { const previous = ready.get(key); if (previous) readyChars -= previous.length; ready.delete(key); } + const cached = ready.get(key); + if (cached) { ready.delete(key); ready.set(key, cached); return Promise.resolve(cached); } const existing = pending.get(key); if (existing) return existing; const loaded = readBytes(sessionId, artifactId) - .then((result) => { - const outcome = decideImageReadOutcome(result); - return outcome.kind === 'image' - ? `data:${outcome.safeMime};base64,${outcome.base64}` - : undefined; + .then(mediaImageSource) + .then(source => { + if (source) { + ready.set(key, source); readyChars += source.length; + while (ready.size > 32 || readyChars > 8 * 1024 * 1024) { + const oldest = ready.keys().next().value!; + readyChars -= ready.get(oldest)!.length; ready.delete(oldest); + } + } + return source; }) .catch(() => undefined); pending.set(key, loaded); @@ -120,7 +130,7 @@ export function useAttachmentImage(ref: { if (!request) return; setResult({ request, status: 'loading' }); let cancelled = false; - request.loadImage(request.sessionId, request.artifactId) + request.loadImage(request.sessionId, request.artifactId, request.attempt > 0) .then((loaded) => { if (!cancelled) setResult({ request, status: loaded ? 'ready' : 'failed', src: loaded }); }); diff --git a/packages/ui/src/index.ts b/packages/ui/src/index.ts index 4a04caf2a19..f995d0a1b01 100644 --- a/packages/ui/src/index.ts +++ b/packages/ui/src/index.ts @@ -206,3 +206,7 @@ export { formatCompactTokenCount } from './compact-token-count.js'; export * from './revision-staged-context.js'; export { TranscriptDisclosure } from './transcript-disclosure.js'; + +export { SessionMediaImage } from './session-media-image.js'; +export { mediaImageSource } from './media-image-source.js'; +export { MediaImageDisplay } from './session-media-image.js'; diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx index df683852d57..dade0d97085 100644 --- a/packages/ui/src/markdown-body.tsx +++ b/packages/ui/src/markdown-body.tsx @@ -30,7 +30,7 @@ */ import { MarkdownImage, createMarkdownImagePlugins } from './markdown-image.js'; -import { useCallback, useContext, useMemo, useRef, type ReactNode } from 'react'; +import { lazy, Suspense, useCallback, useContext, useMemo, useRef, type ReactNode } from 'react'; import { Markdown as AstryxMarkdown, type MarkdownComponents, @@ -53,6 +53,8 @@ import { prepareMarkdownMath, } from './markdown-math.js'; +const SvgCodePreview = lazy(() => import('./svg-code-preview.js')); + const BASE_MARKDOWN_COMPONENTS = { link: MarkdownLink, image: MarkdownImage, @@ -64,15 +66,22 @@ export const MAX_AUTOMATIC_MERMAID_SOURCE_LENGTH = 4_000; export const MAX_AUTOMATIC_MERMAID_TOTAL_SOURCE_LENGTH = 8_000; const CODE_BLOCK_COLLAPSIBLE_THRESHOLD = 10; const DEFERRED_MERMAID_LANGUAGE = 'makamermaiddeferred'; +const DEFERRED_SVG_LANGUAGE = 'makasvgdeferred'; +export const MAX_AUTOMATIC_SVG_DIAGRAMS = 3; +export const MAX_AUTOMATIC_SVG_SOURCE_LENGTH = 64 * 1024; /** - * Mark settled Mermaid fences that exceed the per-document automatic-render + * Mark settled diagram fences that exceed the per-document automatic-render * budget. The private language marker survives Astryx parsing without changing * the source shown to the user; the code renderer turns it into an explicit * source + Render action instead of scheduling more main-thread layout work. */ export function applyMermaidRenderBudget(source: string): string { + // Most messages contain no diagrams: avoid allocating a line array on the hot path. + if (!/mermaid|svg/i.test(source)) return source; const lines = source.split('\n'); + let svgCount = 0; + let svgLength = 0; let automaticCount = 0; let automaticSourceLength = 0; @@ -80,8 +89,7 @@ export function applyMermaidRenderBudget(source: string): string { const opening = /^( {0,3})(`{3,}|~{3,})([^\n]*)$/.exec(lines[index] ?? ''); if (!opening) continue; const [, indent = '', fence = '', info = ''] = opening; - const language = /^([ \t]*)mermaid(?=[ \t]|$)/i.exec(info); - if (!language) continue; + const language = /^([ \t]*)(mermaid|svg)(?=[ \t]|$)/i.exec(info); const fenceCharacter = fence[0]; if (!fenceCharacter) continue; @@ -90,20 +98,23 @@ export function applyMermaidRenderBudget(source: string): string { while (closingIndex < lines.length && !closing.test(lines[closingIndex] ?? '')) { closingIndex += 1; } - if (closingIndex >= lines.length) continue; + if (closingIndex >= lines.length) break; + if (!language) { index = closingIndex; continue; } const codeLength = lines.slice(index + 1, closingIndex).join('\n').length; - const withinBudget = + const svg = language[2]?.toLowerCase() === 'svg'; + const withinBudget = svg ? codeLength <= MAX_AUTOMATIC_SVG_SOURCE_LENGTH && svgCount < MAX_AUTOMATIC_SVG_DIAGRAMS + && svgLength + codeLength <= 2 * MAX_AUTOMATIC_SVG_SOURCE_LENGTH : codeLength <= MAX_AUTOMATIC_MERMAID_SOURCE_LENGTH && automaticCount < MAX_AUTOMATIC_MERMAID_DIAGRAMS && automaticSourceLength + codeLength <= MAX_AUTOMATIC_MERMAID_TOTAL_SOURCE_LENGTH; if (withinBudget) { - automaticCount += 1; - automaticSourceLength += codeLength; + if (svg) { svgCount++; svgLength += codeLength; } + else { automaticCount += 1; automaticSourceLength += codeLength; } } else { const leadingWhitespace = language[1] ?? ''; - lines[index] = `${indent}${fence}${leadingWhitespace}${DEFERRED_MERMAID_LANGUAGE}${info.slice(language[0].length)}`; + lines[index] = `${indent}${fence}${leadingWhitespace}${svg ? DEFERRED_SVG_LANGUAGE : DEFERRED_MERMAID_LANGUAGE}${info.slice(language[0].length)}`; } index = closingIndex; } @@ -208,30 +219,30 @@ export function MarkdownBody(props: { } function MarkdownCodeDefault(props: { code: string; language?: string }) { - return ; + return ; } function MarkdownCodeCompact(props: { code: string; language?: string }) { - return ; + return ; } function MarkdownCodeStreamingDefault(props: { code: string; language?: string }) { - return ; + return ; } function MarkdownCodeStreamingCompact(props: { code: string; language?: string }) { - return ; + return ; } function MarkdownCode(props: { code: string; language?: string; density: 'default' | 'compact'; - renderMermaid: boolean; + renderDiagrams: boolean; }) { const t = useTranslator(); const language = props.language?.trim().toLowerCase(); - if (props.renderMermaid && (language === 'mermaid' || language === DEFERRED_MERMAID_LANGUAGE)) { + if (props.renderDiagrams && (language === 'mermaid' || language === DEFERRED_MERMAID_LANGUAGE)) { return ( }>; + } + const codeLines = props.code.split('\n'); if (codeLines.length > 1 && codeLines.at(-1) === '') codeLines.pop(); const isSingleLine = codeLines.length === 1; diff --git a/packages/ui/src/markdown-image.tsx b/packages/ui/src/markdown-image.tsx index 5b66689ad34..59ef00081af 100644 --- a/packages/ui/src/markdown-image.tsx +++ b/packages/ui/src/markdown-image.tsx @@ -17,18 +17,13 @@ * under the License. */ -import { useEffect, useRef, useState } from 'react'; -import { Button, IconButton, Spinner, Tooltip } from '@astryxdesign/core'; import { createMarkdownPlugin, type MarkdownExtensionNode } from '@astryxdesign/core/Markdown/plugins'; import type { MarkdownAstNode, MarkdownAstRoot } from '@astryxdesign/core/Markdown'; -import { RotateCw, AlertTriangle } from './icons.js'; -import { Link } from '@astryxdesign/core/Link'; -import { useChatImageResource } from './chat-image-resource.js'; -import { getSharedUiCopy } from './shared-ui-copy.js'; -import { useUiLocale } from './locale-context.js'; import { createMarkdownImageSourceResolver } from './markdown-image-source.js'; import { redactSecrets } from './redact.js'; +import { SessionMediaImage as MarkdownImage } from './session-media-image.js'; + type PlacedImage = MarkdownExtensionNode<'maka-images', 'image', { src: string; alt: string; inline: boolean; redacted: boolean }>; // Markdown structure determines inline/block placement; decoded images supply // their natural dimensions within the presentation's CSS bounds. @@ -78,54 +73,5 @@ function isSingleImage(nodes: readonly MarkdownAstNode[]): boolean { return node.type === 'image' || (['link', 'strong', 'emphasis', 'delete'].includes(node.type) && 'children' in node && isSingleImage(node.children)); } -/** Presentation only: Host resolves local addresses and archives; UI receives artifact identities. */ -export function MarkdownImage(props: { src: string; alt: string; inline?: boolean; redacted?: boolean }) { - return ; -} -function ImageResource(props: { src: string; redacted: boolean; alt: string; inline?: boolean }) { - const copy = getSharedUiCopy(useUiLocale()).markdown; - const anchor = useRef(null); - const [visible, setVisible] = useState(typeof IntersectionObserver === 'undefined'); - useEffect(() => { - if (visible || !anchor.current) return; - const observer = new IntersectionObserver(entries => { - if (entries.some(entry => entry.isIntersecting)) { setVisible(true); observer.disconnect(); } - }, { rootMargin: '300px' }); - observer.observe(anchor.current); return () => observer.disconnect(); - }, [visible]); - const resource = useChatImageResource({ source: props.src, redacted: props.redacted, visible }); - const state = resource.presentation; - const message = state.kind === 'loading' ? copy.imageLoading - : state.kind === 'failed' ? state.reason === 'saved_bytes' ? copy.imageLoadFailed : copy.imageArchiveFailure(state.reason) - : state.kind === 'unavailable' ? { - attachment: copy.imageUnavailable, - remote: copy.imageRemoteUnavailable, - redacted: copy.imageRemoteRedacted, - unsupported: copy.imageUnsupported, - }[state.reason] : undefined; - const actions = - {resource.retry &&