diff --git a/apps/desktop/src/renderer/features/conversation/staging-services.ts b/apps/desktop/src/renderer/features/conversation/staging-services.ts index 9cbfb55c767..ac3880df391 100644 --- a/apps/desktop/src/renderer/features/conversation/staging-services.ts +++ b/apps/desktop/src/renderer/features/conversation/staging-services.ts @@ -25,6 +25,7 @@ import type { ResolveImageDelivery } from '@maka/core/image-delivery'; /** One attachment port: the Composer stages through it and the transcript reads image bytes through it. */ export interface ComposerStagingServices extends ComposerAttachmentService { readBytes(sessionId: string, artifactId: string): Promise; + openSavedImage?: (sessionId: string, artifactId: string) => Promise<{ ok: boolean }>; resolveImageDelivery?: ResolveImageDelivery; } const context = createServicesContext('ComposerStagingServicesProvider'); diff --git a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx index 78366070d5e..8fd1d47d2a1 100644 --- a/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx +++ b/apps/desktop/src/renderer/features/conversation/ui/staged-quote-chat-view.tsx @@ -24,8 +24,8 @@ import { useComposerStagingServices } from '../staging-services.js'; /** Transcript attachment reads come from the injected attachment port, never from the caller. */ export function StagedQuoteChatView(props: Omit, - 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' | 'onResolveImageDelivery'>) { + 'handleRef' | 'pendingQuotes' | 'onQuoteAnnotationSubmit' | 'onReadAttachmentBytes' | 'onResolveImageDelivery' | 'onOpenSavedImage'>) { const staging = useComposerStaging(); - const { readBytes, resolveImageDelivery } = useComposerStagingServices(); - return ; + const { readBytes, resolveImageDelivery, openSavedImage } = useComposerStagingServices(); + return ; } diff --git a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx index 0a4b8e8af21..c931057898f 100644 --- a/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx +++ b/apps/desktop/src/renderer/features/workbar/tools/side-chat/quote-companion-panel.tsx @@ -86,7 +86,7 @@ export function QuoteCompanionPanel(props: { onPromptAccepted?: (panelId: string, prompt: string) => void; onActivityStateChange?: (panelId: string, active: boolean) => void; }) { - const { attachments } = useWorkbarServices(); + const { attachments, artifacts } = useWorkbarServices(); const mentions = useComposerMentionsContext(); const locale = useUiLocale(); const toast = useToast(); @@ -404,7 +404,7 @@ export function QuoteCompanionPanel(props: { liveTurns={companion.liveTurns} activeTurn={companion.activeTurn} activeSession={companion.companionSession} - onReadAttachmentBytes={attachments.readBytes} onResolveImageDelivery={attachments.resolveImageDelivery} + onOpenSavedImage={artifacts.openPath} onReadAttachmentBytes={attachments.readBytes} onResolveImageDelivery={attachments.resolveImageDelivery} deriveTurnPresentation={deriveTurnPresentation} onEditUserMessage={(turnId) => { const message = companion.messages.find( diff --git a/apps/desktop/src/renderer/features/workhub/ports.ts b/apps/desktop/src/renderer/features/workhub/ports.ts index 5b7faf32569..c9831f4adfb 100644 --- a/apps/desktop/src/renderer/features/workhub/ports.ts +++ b/apps/desktop/src/renderer/features/workhub/ports.ts @@ -58,6 +58,7 @@ export interface WorkHubServices extends WorkHubWorkspaceServices { model: string; }): Promise; readonly attachments: ComposerAttachmentService; + openSavedImage?: (sessionId: string, artifactId: string) => Promise<{ ok: boolean }>; readAttachmentBytes(sessionId: string, artifactId: string): Promise; resolveImageDelivery?: ResolveImageDelivery; prepareAttachments(sessionId: string, items: Array<{ approvalId: string; name: string; mimeType?: string } | { file: File }>): Promise; diff --git a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx index cf4523f19b8..cd58397c3ae 100644 --- a/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx +++ b/apps/desktop/src/renderer/features/workhub/ui/workhub-root.tsx @@ -412,7 +412,7 @@ export function WorkHubRoot() {
call(services.presentation.openSession(id))} scrollBehavior="auto" onNew={() => composer.current?.focus()} diff --git a/apps/desktop/src/renderer/platform/desktop/create-composer-staging-services.ts b/apps/desktop/src/renderer/platform/desktop/create-composer-staging-services.ts index 2c263ce7986..221e062f39a 100644 --- a/apps/desktop/src/renderer/platform/desktop/create-composer-staging-services.ts +++ b/apps/desktop/src/renderer/platform/desktop/create-composer-staging-services.ts @@ -21,7 +21,7 @@ import type { MakaBridge } from '../../../preload/bridge-contract.js'; import type { ComposerStagingServices } from '../../features/conversation/index.js'; export function createDesktopComposerStagingServices( - bridge: Pick = window.maka, + bridge: Pick = window.maka, ): ComposerStagingServices { - return bridge.attachments; + return { ...bridge.attachments, openSavedImage: (sessionId, artifactId) => bridge.app.openArtifactPath(sessionId, artifactId) }; } diff --git a/apps/desktop/src/renderer/platform/desktop/create-workhub-services.ts b/apps/desktop/src/renderer/platform/desktop/create-workhub-services.ts index c5ab42c44b5..6500908b17b 100644 --- a/apps/desktop/src/renderer/platform/desktop/create-workhub-services.ts +++ b/apps/desktop/src/renderer/platform/desktop/create-workhub-services.ts @@ -47,6 +47,7 @@ export function createDesktopWorkHubServices( | 'runtimeHostProfiles' | 'settings' | 'attachments' + | 'app' > = window.maka, ): WorkHubServices { return { @@ -93,6 +94,7 @@ export function createDesktopWorkHubServices( bridge.connections.setDefaultModel({ slug: llmConnectionSlug, model }), attachments: bridge.attachments, readAttachmentBytes: bridge.attachments.readBytes, + openSavedImage: (sessionId, artifactId) => bridge.app.openArtifactPath(sessionId, artifactId), resolveImageDelivery: bridge.attachments.resolveImageDelivery, prepareAttachments: async (sessionId, items) => { const result = await bridge.workHub.prepareAttachments(sessionId, items); diff --git a/docs/architecture/session-media-preview.zh-CN.md b/docs/architecture/session-media-preview.zh-CN.md new file mode 100644 index 00000000000..74b10575cc4 --- /dev/null +++ b/docs/architecture/session-media-preview.zh-CN.md @@ -0,0 +1,44 @@ + + +# ACP 会话媒体发布 + +ACP 的 image、resource_link 和 embedded resource 通过 `PluginExecutorContext.publishMedia` 交给 Host，复用 `ChatImageDeliveryService` 的读取边界、联网策略、校验、队列、配额和 Artifact 保存。媒体格式由 Host 校验器决定；本变更沿用 PNG/JPEG/GIF/WebP 支持，SVG 格式与预览由独立变更承接。图片输出只携带当前 Session 的 Artifact 引用，不把 base64 写进消息事件。每次路径或 URL 发布独立捕获，避免同一轮覆盖源文件后复用旧图；已保存结果始终按 Artifact 引用回放，内联内容仍按字节去重。 + +正文结构化图片转为 `maka://runtime/attachments/` Markdown 引用。工具结果新增有序 `content.parts`，支持文本、已保存图片和 diff；旧 text/file_diff/image 保持可读。旧客户端无法解码新结果，兼容 epoch 从 229 升至 230，合并前需对照 main 确认。 + +通用复制层收集并重写混合内容的图片引用；ACP 会话仍受既有 provider-owned conversation cloning 限制，本变更不开放外部执行器会话复制。Goal 评估递归保留混合结果中的文字和 diff，省略图片，并沿用脱敏和长度限制。CLI 按顺序输出描述；没有媒体发布能力或发布失败时返回有界文字提示。 + +通知接收时完成准入并绑定当前轮次，串行处理以保留到达顺序。取消或断连中断媒体等待，但收尾已收到的文字和工具结果；prompt 返回后仍检测断连直至队列排空，迟到通知和媒体完成不会写回已结束轮次。慢图片归档会延迟后续文字，应在端到端验收中观察。 + +Host 最多 2 个并行捕获、128 个内存任务、512 个持久接收任务；解码中的内联字节最多 8 MiB，SQLite 暂存字节最多 32 MiB。有效原图接收上限为 8 MiB，预览仍限 2 MiB 和既有像素约束；超预览大小的已保存图片提供打开原件入口。ACP 每轮序列化媒体预算为 12 MiB，以容纳 8 MiB 原图的 base64 扩展。ACP 按唯一媒体计数和累计字节限制每轮保留内容，预算按裁剪后媒体对象的序列化 UTF-8 字节计费，audio 计入预算；未使用的媒体元数据、非文本 thought 和不展示的 user message 通知在入队前丢弃。重复工具快照复用媒体对象；纯文本和纯 diff 沿用旧规则，混合结果最多 64 项，超出明确提示省略。 + +验证覆盖媒体投影、真实 MIME、配额和网络失败、源文件删除后的归档回放、会话复制，以及取消和断连时的顺序与收尾。SVG 文件和代码块预览独立验收。 + +## 接收、保存与恢复 + +正文和 ACP 结构化媒体使用同一 Host 接收服务。已提交正文中的授权本地/远程图片主动登记持久任务，接收不再依赖图片是否滚入视口；预览字节仍按需读取。任务与 pending 状态原子写入 Artifact SQLite；已收到的字节先以有界 BLOB 暂存及 SHA-256 校验，再进入既有 fsync 文件和元数据提交流程。最终消息仍只存引用，最终图片仍存 Artifact 文件。 + +临时读取失败自动重试至多三次，次数与下次时间持久化；权限、格式、尺寸等永久失败退出队列，只保留可见失败原因。配额失败的有效暂存字节可供手动重试，但在新工作需要容量时按旧任务顺序回收；失败状态仍保留。接收缓冲区不是超出归档配额的永久文件仓库。删除会话同时清理任务和暂存字节。 + +消息正文结束与图片接收完成分别展示，不能用正文完成代表所有附件已保存。待接收时展示接收状态；已保存但超过预览上限时展示原件入口；真正缺失、被拒绝或超出归档配额的来源仍显示失败及可用的重试操作，不伪装成功。 + +启动恢复在中断运行封口前先完成已暂存字节的归档。ACP 图片的 Artifact 元数据保留准确 publicationRunId；如果已归档图片尚未进入该运行的 canonical 正文或工具结果，Host 写入一条明确标注的恢复图片正文，且对模型历史隐藏。确定性事件 ID 和 Artifact 创建时间保证再次崩溃后可幂等回放，不伪造原工具完成状态或顺序。已结束运行不重新打开；取消后晚到且未进入历史的图片不通过此路径追加。对于正文提交后尚未来得及登记任务的中断运行，启动补扫已提交正文恢复登记。 + +接收意图不等于已经收到字节：在首次源读取前消失的文件/URL无法凭空恢复；超过配额、磁盘写入失败和非法图片也不能承诺保存成功。已收到字节的进程崩溃恢复与源消失、已归档但尚未发布的消息恢复分别测试。 diff --git a/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts b/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts index f4c383b8e94..b836906c837 100644 --- a/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts +++ b/packages/acp-executor-plugin/src/__tests__/acp-executor-plugin.test.ts @@ -2971,6 +2971,689 @@ test('questions retain option identity and output updates retain arrival order', } }); +test('ACP waits for asynchronous media publication and preserves surrounding text', async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + const events: unknown[] = []; + try { + const result = await executor.execute(request('media'), { + ...executorContext(events), + requestPermission: async () => { + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'before' }, + }); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'image', mimeType: 'image/png', data: 'AAAA' }, + }); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'after' }, + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + await new Promise((resolve) => setTimeout(resolve, 5)); + return { + kind: 'image', + mimeType: 'image/png', + ref: { kind: 'session_file', sessionId: 'session-a', relativePath: 'saved-image' }, + }; + }, + }); + assert.equal(result.status, 'completed'); + if (result.status !== 'completed') throw new Error('Expected completion'); + assert.equal( + result.text, + 'before\n\n![](maka://runtime/attachments/saved-image)\n\nafterreply:media', + ); + assert.equal( + events + .filter((e: any) => e.type === 'output_delta') + .map((e: any) => e.text) + .join(''), + result.text, + ); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +for (const placement of ['message', 'tool'] as const) { + test(`cancelling ACP drains ${placement} updates without waiting for media publication`, async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const storage = durableState(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { + createConnection: protocol.factory, + state: storage.state, + }, + ); + const abort = new AbortController(); + const media = deferred(); + const started = deferred(); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + let publications = 0; + const execution = executor.execute(request('cancel-media'), { + ...executorContext(events, abort.signal), + requestPermission: async () => { + const content = { type: 'image', mimeType: 'image/png', data: 'AAAA' }; + protocol.notifyUpdate( + placement === 'message' + ? { sessionUpdate: 'agent_message_chunk', content } + : { + sessionUpdate: 'tool_call', + toolCallId: 'draw', + title: 'Draw', + status: 'completed', + content: [{ type: 'content', content }], + }, + ); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'after image' }, + }); + protocol.notifyUpdate({ sessionUpdate: 'agent_message_chunk', content }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: () => { + publications++; + started.resolve(); + return media.promise; + }, + }); + let timeout: ReturnType | undefined; + try { + await started.promise; + abort.abort(new Error('user_stop')); + const result = await Promise.race([ + execution, + new Promise((resolve) => { + timeout = setTimeout(() => resolve(undefined), 1000); + }), + ]); + assert.equal(result?.status, 'cancelled', 'Cancellation must not wait for Host media'); + assert.equal(publications, 1, 'Queued media must not start after cancellation'); + assert.ok( + events.some((event) => event.type === 'output_delta' && event.text === 'after image'), + ); + if (placement === 'tool') + assert.ok( + events.some((event) => event.type === 'tool_result' && event.toolCallId === 'draw'), + ); + await executor.acknowledgeExecution('session-a', 'turn-cancel-media'); + assert.equal(storage.record().phase, 'committed'); + assert.equal( + (await executor.execute(request('next'), executorContext(events))).status, + 'completed', + ); + const settledEvents = structuredClone(events); + // A late Host failure must be observed without reopening the completed turn. + media.reject(new Error('late image failure')); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(events, settledEvents); + } finally { + clearTimeout(timeout); + media.resolve({ kind: 'text', text: 'released' }); + await execution; + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } + }); +} + +for (const placement of ['message', 'tool'] as const) { + for (const promptState of ['pending', 'returned'] as const) { + test(`ACP transport failure drains received ${placement} updates with ${promptState} prompt and rejects late updates`, async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const crashed = deferred(); + const prompt = deferred<{ stopReason: 'end_turn' }>(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { + createConnection: (input) => { + const owner = protocol.factory(input); + const request = owner.connection.agent.request.bind(owner.connection.agent); + owner.connection.agent.request = (async (method: string, params: never) => { + if (method === methods.agent.session.prompt) { + const content = { type: 'image', mimeType: 'image/png', data: 'AAAA' }; + protocol.notifyUpdate( + placement === 'message' + ? { sessionUpdate: 'agent_message_chunk', content } + : { + sessionUpdate: 'tool_call', + toolCallId: 'draw', + title: 'Draw', + status: 'completed', + content: [{ type: 'content', content }], + }, + ); + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'after image' }, + }); + protocol.notifyUpdate({ + sessionUpdate: 'tool_call', + toolCallId: 'after', + title: 'After', + status: 'completed', + content: [ + { type: 'content', content: { type: 'text', text: 'received result' } }, + ], + }); + return prompt.promise; + } + return request(method as never, params); + }) as typeof owner.connection.agent.request; + return { + ...owner, + failed: crashed.promise, + dispose: async () => { + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: 'late text' }, + }); + await owner.dispose(); + }, + }; + }, + }, + ); + const media = deferred(); + const started = deferred(); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = + []; + const execution = executor.execute(request('crash-media'), { + ...executorContext(events), + publishMedia: () => { + started.resolve(); + return media.promise; + }, + }); + let timeout: ReturnType | undefined; + try { + await started.promise; + if (promptState === 'returned') { + prompt.resolve({ stopReason: 'end_turn' }); + await new Promise((resolve) => setImmediate(resolve)); + } + crashed.reject(new Error('ACP process exited')); + const result = await Promise.race([ + execution, + new Promise((resolve) => { + timeout = setTimeout(() => resolve(undefined), 1000); + }), + ]); + assert.equal(result?.status, 'failed', 'Transport failure must not wait for Host media'); + assert.ok( + events.some((event) => event.type === 'output_delta' && event.text === 'after image'), + 'Already-received text must survive transport failure', + ); + assert.ok( + events.some((event) => event.type === 'tool_result' && event.toolCallId === 'after'), + 'Already-received tool results must survive transport failure', + ); + if (placement === 'tool') + assert.ok( + events.some((event) => event.type === 'tool_result' && event.toolCallId === 'draw'), + ); + assert.ok( + !events.some((event) => event.type === 'output_delta' && event.text === 'late text'), + ); + const settledEvents = structuredClone(events); + media.reject(new Error('late image failure')); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(events, settledEvents); + } finally { + clearTimeout(timeout); + media.resolve({ kind: 'text', text: 'released' }); + prompt.resolve({ stopReason: 'end_turn' }); + await execution; + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } + }); + } +} + +test('repeated ACP tool snapshots retain their image without exhausting the media budget', async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + let publications = 0; + try { + const result = await executor.execute(request('repeated-image'), { + ...executorContext(events), + requestPermission: async () => { + // Each notification is a fresh protocol snapshot containing the same 1.5 MiB image. + for (let i = 0; i < 70; i++) + protocol.notifyUpdate({ + sessionUpdate: i === 0 ? 'tool_call' : 'tool_call_update', + toolCallId: 'draw', + title: 'Draw', + status: i === 69 ? 'completed' : 'in_progress', + content: [ + { + type: 'content', + content: { + type: 'image', + mimeType: 'image/png', + data: 'A'.repeat(2 * 1024 * 1024), + }, + }, + ], + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + publications++; + return { + kind: 'image', + mimeType: 'image/png', + ref: { kind: 'session_file', sessionId: 'session-a', relativePath: 'saved-image' }, + }; + }, + }); + assert.equal(result.status, 'completed'); + assert.equal(publications, 1); + const output = events.find( + (event) => event.type === 'tool_result' && event.toolCallId === 'draw', + ); + assert.ok(output?.type === 'tool_result' && output.content.kind === 'content'); + assert.equal(output.content.parts[0]?.kind, 'image'); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); +test('a seven MiB ACP image reaches Host publication', async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + let publications = 0; + try { + const result = await executor.execute(request('repeated-image'), { + ...executorContext(events), + requestPermission: async () => { + // Base64 expansion must not make a valid original exceed the wire budget. + for (let i = 0; i < 1; i++) + protocol.notifyUpdate({ + sessionUpdate: i === 0 ? 'tool_call' : 'tool_call_update', + toolCallId: 'draw', + title: 'Draw', + status: i === 0 ? 'completed' : 'in_progress', + content: [ + { + type: 'content', + content: { + type: 'image', + mimeType: 'image/png', + data: Buffer.alloc(7 * 1024 * 1024).toString('base64'), + }, + }, + ], + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + publications++; + return { + kind: 'image', + mimeType: 'image/png', + ref: { kind: 'session_file', sessionId: 'session-a', relativePath: 'saved-image' }, + }; + }, + }); + assert.equal(result.status, 'completed'); + assert.equal(publications, 1); + const output = events.find( + (event) => event.type === 'tool_result' && event.toolCallId === 'draw', + ); + assert.ok(output?.type === 'tool_result' && output.content.kind === 'content'); + assert.equal(output.content.parts[0]?.kind, 'image'); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +for (const type of ['image', 'audio'] as const) { + for (const placement of ['message', 'tool'] as const) { + for (const limit of ['count', 'bytes', 'single-block'] as const) { + test(`unique ACP ${type} ${placement} media still respects the ${limit} limit`, async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + let publications = 0; + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = + []; + try { + const result = await executor.execute(request('media-limit'), { + ...executorContext(events), + requestPermission: async () => { + const count = limit === 'count' ? 65 : limit === 'bytes' ? 7 : 1; + for (let i = 0; i < count; i++) { + const content = { + type, + mimeType: type === 'image' ? 'image/png' : 'audio/wav', + data: Buffer.alloc( + limit === 'count' ? 3 : limit === 'bytes' ? 1.5 * 1024 * 1024 : 9 * 1024 * 1024, + i, + ).toString('base64'), + }; + protocol.notifyUpdate( + placement === 'message' + ? { + sessionUpdate: 'agent_message_chunk', + content, + } + : { + sessionUpdate: 'tool_call', + toolCallId: `media-${i}`, + title: 'Media', + status: 'completed', + content: [{ type: 'content', content }], + }, + ); + } + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + publications++; + return { kind: 'text', text: 'saved' }; + }, + }); + assert.equal( + publications, + type === 'audio' ? 0 : limit === 'count' ? 64 : limit === 'bytes' ? 5 : 0, + ); + assert.ok(result.status === 'completed'); + assert.match( + placement === 'message' + ? result.text + : JSON.stringify(events.filter((event) => event.type === 'tool_result')), + /Media omitted: response media limit reached/, + ); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } + }); + } + } +} + +for (const kind of ['text', 'diff'] as const) { + test(`ACP preserves all small ${kind} blocks beyond the mixed-content part limit`, async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { createConnection: protocol.factory }, + ); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + const values = Array.from({ length: 65 }, (_, i) => `result-${i}`); + try { + await executor.execute(request(`many-${kind}`), { + ...executorContext(events), + requestPermission: async () => { + protocol.notifyUpdate({ + sessionUpdate: 'tool_call', + toolCallId: 'many', + title: 'Many results', + status: 'completed', + content: + kind === 'text' + ? values.map((text) => ({ type: 'content', content: { type: 'text', text } })) + : values.map((path) => ({ type: 'diff', path, oldText: '', newText: 'new' })), + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + }); + const result = events.find((event) => event.type === 'tool_result'); + assert.ok(result?.type === 'tool_result'); + assert.deepEqual(result.content, { + kind: 'text', + text: + kind === 'text' + ? values.join('\n') + : `${values.map((path) => `Updated ${path}`).join('\n')}\nDiff omitted because it exceeds the executor event limit.`, + }); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } + }); +} + +for (const placement of ['first', 'last', 'replacement', 'status-only'] as const) { + test(`ACP bounds mixed tool content and reports omissions with ${placement} media`, async () => { + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { + createConnection: protocol.factory, + }, + ); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + let publications = 0; + try { + await executor.execute(request('many-parts'), { + ...executorContext(events), + requestPermission: async () => { + const image = { + type: 'content', + content: { type: 'image', mimeType: 'image/png', data: 'AAAA' }, + }; + const content = Array.from({ length: 65 }, (_, i) => ({ + type: 'diff', + path: `file-${i}`, + oldText: '', + newText: 'new', + })); + protocol.notifyUpdate({ + sessionUpdate: 'tool_call', + toolCallId: 'many', + title: 'Many parts', + status: + placement === 'replacement' || placement === 'status-only' + ? 'in_progress' + : 'completed', + content: placement === 'last' ? [...content, image] : [image, ...content], + }); + if (placement === 'replacement') + protocol.notifyUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: 'many', + status: 'completed', + content: [{ type: 'content', content: { type: 'text', text: 'replacement' } }], + }); + if (placement === 'status-only') + protocol.notifyUpdate({ + sessionUpdate: 'tool_call_update', + toolCallId: 'many', + status: 'completed', + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: async () => { + publications++; + return { + kind: 'image', + mimeType: 'image/png', + ref: { + kind: 'session_file', + sessionId: 'session-a', + relativePath: 'saved-image', + }, + }; + }, + }); + const result = events.find((event) => event.type === 'tool_result'); + assert.ok(result?.type === 'tool_result'); + if (placement === 'replacement') { + assert.deepEqual(result.content, { kind: 'text', text: 'replacement' }); + } else { + assert.equal(result.content.kind, 'content'); + if (result.content.kind !== 'content') throw new Error('Expected mixed result'); + assert.equal(result.content.parts.length, 64); + assert.deepEqual(result.content.parts.at(-1), { + kind: 'text', + text: '[Additional tool content omitted]', + }); + assert.equal(result.content.parts[0]?.kind, placement === 'last' ? 'file_diff' : 'image'); + } + assert.equal(publications, placement === 'first' || placement === 'status-only' ? 1 : 0); + } finally { + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } + }); +} + +test('ACP discards ignored notifications and unused media metadata before the blocked publication queue', async () => { + if (!global.gc) { + const { execFile } = await import('node:child_process'); + const { promisify } = await import('node:util'); + const { fileURLToPath } = await import('node:url'); + const env = { ...process.env }; + delete env.NODE_TEST_CONTEXT; + const { stdout } = await promisify(execFile)( + process.execPath, + [ + '--expose-gc', + '--test', + '--test-reporter=tap', + '--test-name-pattern=^ACP discards ignored notifications and unused media metadata before the blocked publication queue$', + fileURLToPath(import.meta.url), + ], + { timeout: 15_000, env }, + ); + assert.match(stdout, /^# pass 1$/m, 'The isolated GC regression must execute'); + return; + } + const fixture = await executableFixture(); + const protocol = fakeProtocol(); + const executor = new AcpExecutor( + adapter, + { executable: fixture.executable }, + { + createConnection: protocol.factory, + }, + ); + const started = deferred(); + const media = deferred(); + const events: import('@maka/runtime/plugin-executor-service').PluginExecutorOutputEvent[] = []; + const publications: import('@maka/core/session-media').SessionMediaInput[] = []; + const execution = executor.execute(request('thought-media'), { + ...executorContext(events), + requestPermission: async () => { + protocol.notifyUpdate({ + sessionUpdate: 'agent_message_chunk', + content: { + type: 'image', + mimeType: 'image/png', + data: 'AAAA', + }, + }); + return { outcome: 'selected', optionId: 'allow_once' }; + }, + publishMedia: (input) => { + publications.push(input); + started.resolve(); + return media.promise; + }, + }); + try { + await started.promise; + const refs: WeakRef[] = Array.from({ length: 65 }, (_, i) => { + const content = { + type: 'audio', + mimeType: 'audio/wav', + data: Buffer.alloc(1024, i).toString('base64'), + }; + protocol.notifyUpdate({ sessionUpdate: 'agent_thought_chunk', content }); + return new WeakRef(content); + }); + refs.push( + ...Array.from({ length: 4 }, (_, i) => { + const content = { + type: 'image', + mimeType: 'image/png', + data: Buffer.alloc(3 * 1024 * 1024, i).toString('base64'), + }; + protocol.notifyUpdate({ sessionUpdate: 'user_message_chunk', content }); + const metadata = { unused: 'x'.repeat(3 * 1024 * 1024) }; + const link = { + type: 'resource_link', + uri: `/tmp/plot-${i}.png`, + name: 'plot', + mimeType: 'image/png', + _meta: metadata, + description: 'x'.repeat(3 * 1024 * 1024), + }; + protocol.notifyUpdate({ sessionUpdate: 'agent_message_chunk', content: link }); + return [new WeakRef(content), new WeakRef(link), new WeakRef(metadata)]; + }).flat(), + ); + let retained = refs.length; + for (let attempt = 0; attempt < 5 && retained; attempt++) { + await new Promise((resolve) => setImmediate(resolve)); + global.gc(); + retained = refs.filter((ref) => ref.deref() !== undefined).length; + } + assert.equal(retained, 0, 'Ignored media must be collectible while publication is blocked'); + protocol.notifyUpdate({ + sessionUpdate: 'agent_thought_chunk', + content: { type: 'text', text: 'kept thought' }, + }); + media.resolve({ kind: 'text', text: 'published' }); + assert.equal((await execution).status, 'completed'); + assert.deepEqual( + publications.slice(1), + Array.from({ length: 4 }, (_, i) => ({ kind: 'source', source: `/tmp/plot-${i}.png` })), + ); + assert.deepEqual( + events.filter((event) => event.type === 'thinking_delta'), + [{ type: 'thinking_delta', text: 'kept thought' }], + ); + } finally { + media.resolve({ kind: 'text', text: 'released' }); + await execution; + await executor.dispose(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); + function request(text: string) { return { sessionId: 'session-a', @@ -3534,11 +4217,13 @@ test('registered-style directory ownership covers Agent startup and cleanup', as }); function deferred() { - let resolve!: (value: T) => void; - const promise = new Promise((done) => { - resolve = done; + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; }); - return { promise, resolve }; + return { promise, resolve, reject }; } test('restoration reasserts the saved mode even when the request only selects a model', async () => { diff --git a/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts b/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts index 1833a7dfa47..c46c882b8a1 100644 --- a/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts +++ b/packages/acp-executor-plugin/src/__tests__/acp-projection.test.ts @@ -49,3 +49,53 @@ for (const { name, oldText, newText, expected } of [ ); }); } + +test('rich ACP content preserves text, SVG resource, image and diff order', async () => { + const { projectRichToolResult } = await import('../acp-projection.js'); + const inputs: unknown[] = []; + const result = await projectRichToolResult( + [ + { type: 'content', content: { type: 'text', text: 'before' } }, + { + type: 'content', + content: { + type: 'resource', + resource: { uri: 'drawing.svg', mimeType: 'image/svg+xml', text: '' }, + }, + }, + { type: 'content', content: { type: 'image', mimeType: 'image/png', data: 'AAAA' } }, + { type: 'diff', path: 'drawing.svg', oldText: '', newText: '' }, + { type: 'content', content: { type: 'text', text: 'after' } }, + ], + undefined, + { + signal: new AbortController().signal, + emit() {}, + requestPermission: async () => ({ outcome: 'cancelled' }), + publishMedia: async (input) => { + inputs.push(input); + return { + kind: 'image', + mimeType: input.kind === 'inline' ? input.mimeType : 'image/png', + ref: { + kind: 'session_file', + sessionId: 'session', + relativePath: `image-${inputs.length}`, + }, + }; + }, + }, + ); + assert.equal(result.kind, 'content'); + if (result.kind !== 'content') throw new Error('Expected content'); + assert.deepEqual( + result.parts.map((part) => part.kind), + ['text', 'image', 'image', 'file_diff', 'text'], + ); + assert.equal(inputs.length, 2); + assert.deepEqual(inputs[0], { + kind: 'inline', + mimeType: 'image/svg+xml', + base64: Buffer.from('').toString('base64'), + }); +}); diff --git a/packages/acp-executor-plugin/src/acp-projection.ts b/packages/acp-executor-plugin/src/acp-projection.ts index 43f73fe96ba..52cfac8d886 100644 --- a/packages/acp-executor-plugin/src/acp-projection.ts +++ b/packages/acp-executor-plugin/src/acp-projection.ts @@ -17,13 +17,21 @@ * under the License. */ -import type { ToolCall, ToolCallContent } from '@agentclientprotocol/sdk'; +import type { ContentBlock, ToolCall, ToolCallContent } from '@agentclientprotocol/sdk'; import type { PluginExecutorContext, PluginExecutorRequest, PluginExecutorToolResultContent, } from '@maka/runtime/plugin-executor-service'; +import { + type SessionContentPart, + type SessionMediaInput, + SESSION_CONTENT_MAX_PARTS, + SESSION_MEDIA_MAX_BYTES, + SESSION_MEDIA_MAX_BASE64, +} from '@maka/core/session-media'; + const MAX_EVENT_TEXT = 8_192; const MAX_TOOL_RESULT_DIFF = 1024 * 1024; @@ -166,3 +174,139 @@ export function activityKind(kind: ToolCall['kind'] | undefined) { if (kind === 'think') return 'explore' as const; return 'tool' as const; } + +/** Preserve protocol content order; only the Host can publish media bytes or read a source. */ +export async function projectContentBlock( + content: ContentBlock, + context: PluginExecutorContext, +): Promise> { + if (content.type === 'text') return { kind: 'text', text: boundedText(content.text) }; + let input: SessionMediaInput | undefined; + if (content.type === 'image') + input = { kind: 'inline', mimeType: content.mimeType, base64: content.data }; + if (content.type === 'resource_link') { + if ( + content.mimeType?.startsWith('image/') || + /\.(svg|png|jpe?g|webp|gif)(?:[?#]|$)/i.test(content.uri) + ) + input = { kind: 'source', source: content.uri }; + else return { kind: 'text', text: boundedText(`${content.name}: ${content.uri}`) }; + } + if (content.type === 'resource') { + const resource = content.resource; + if ('text' in resource) { + if (resource.mimeType === 'image/svg+xml' || /\.svg(?:[?#]|$)/i.test(resource.uri)) { + if (resource.text.length > SESSION_MEDIA_MAX_BYTES) + return { kind: 'text', text: '[Media unavailable: too large]' }; + input = { + kind: 'inline', + mimeType: 'image/svg+xml', + base64: Buffer.from(resource.text).toString('base64'), + }; + } else return { kind: 'text', text: boundedText(resource.text) }; + } else if (resource.mimeType?.startsWith('image/')) + input = { kind: 'inline', mimeType: resource.mimeType, base64: resource.blob }; + } + if (!input || !context.publishMedia) return { kind: 'text', text: '[Media preview unavailable]' }; + if (context.signal.aborted) return { kind: 'text', text: '[Media publication cancelled]' }; + let onAbort!: () => void; + const cancelled = new Promise((resolve) => { + onAbort = () => resolve({ kind: 'text', text: '[Media publication cancelled]' }); + context.signal.addEventListener('abort', onAbort, { once: true }); + }); + try { + // The Host owns archival work. Stop waiting on cancellation, but keep observing late + // rejection through the race and never let late publication emit another ACP update. + const part = await Promise.race([context.publishMedia(input), cancelled]); + return part.kind === 'file_diff' ? { kind: 'text', text: '[Media preview unavailable]' } : part; + } catch { + return { kind: 'text', text: '[Media publication failed]' }; + } finally { + context.signal.removeEventListener('abort', onAbort); + } +} + +export async function projectRichToolResult( + content: readonly ToolCallContent[], + rawOutput: unknown, + context: PluginExecutorContext, + truncated = false, +): Promise { + // The queue already discarded the tail. Preserve its omission even when that + // tail contained the only media and the retained prefix is entirely text/diff. + if (truncated) + content = [ + ...content, + { + type: 'content', + content: { type: 'text', text: '[Additional tool content omitted]' }, + }, + ]; + if ( + !truncated && + !content.some((item) => item.type === 'content' && item.content.type !== 'text') + ) + return projectToolResult(content, rawOutput); + const parts: SessionContentPart[] = []; + let budget = MAX_TOOL_RESULT_DIFF; + for (const item of content.slice(0, SESSION_CONTENT_MAX_PARTS)) { + const part = + item.type === 'content' + ? await projectContentBlock(item.content, context) + : item.type === 'diff' + ? projectToolResult([item], undefined) + : { kind: 'text' as const, text: boundedText(`Terminal ${item.terminalId}`) }; + if (part.kind === 'content') continue; + const normalized: SessionContentPart = + part.kind === 'file_diff' ? { ...part, paths: [...part.paths] } : part; + budget -= Buffer.byteLength(JSON.stringify(normalized), 'utf8'); + if ( + budget < 1024 || + (parts.length === SESSION_CONTENT_MAX_PARTS - 1 && content.length > SESSION_CONTENT_MAX_PARTS) + ) { + parts.push({ kind: 'text', text: '[Additional tool content omitted]' }); + break; + } + parts.push(normalized); + } + return { kind: 'content', parts }; +} + +/** Retain only fields used by projection; provider metadata is never queue-owned. */ +export function normalizeQueuedContent(block: ContentBlock): ContentBlock { + const omitted: ContentBlock = { + type: 'text', + text: '[Media omitted: response media limit reached]', + }; + if (block.type === 'text') return { type: 'text', text: block.text }; + if (block.type === 'image' || block.type === 'audio') { + if (block.data.length > SESSION_MEDIA_MAX_BASE64 || block.mimeType.length > 128) return omitted; + return { type: block.type, data: block.data, mimeType: block.mimeType }; + } + if (block.type === 'resource_link') { + if ( + block.uri.length > 4096 || + block.name.length > MAX_EVENT_TEXT || + (block.mimeType?.length ?? 0) > 128 + ) + return omitted; + return { + type: 'resource_link', + uri: block.uri, + name: block.name, + ...(block.mimeType ? { mimeType: block.mimeType } : {}), + }; + } + const resource = block.resource; + if (resource.uri.length > 4096 || (resource.mimeType?.length ?? 0) > 128) return omitted; + const payload = 'text' in resource ? resource.text : resource.blob; + if (payload.length > SESSION_MEDIA_MAX_BASE64) return omitted; + return { + type: 'resource', + resource: { + uri: resource.uri, + ...(resource.mimeType ? { mimeType: resource.mimeType } : {}), + ...('text' in resource ? { text: resource.text } : { blob: resource.blob }), + }, + }; +} diff --git a/packages/acp-executor-plugin/src/index.ts b/packages/acp-executor-plugin/src/index.ts index 9b86248bd27..48ed34d0485 100644 --- a/packages/acp-executor-plugin/src/index.ts +++ b/packages/acp-executor-plugin/src/index.ts @@ -17,6 +17,7 @@ * under the License. */ +import { sessionContentMarkdown, SESSION_CONTENT_MAX_PARTS } from '@maka/core/session-media'; import { createHash } from 'node:crypto'; import { constants, createReadStream } from 'node:fs'; import { access, mkdir, mkdtemp, realpath, rm, stat } from 'node:fs/promises'; @@ -34,6 +35,7 @@ import { methods, type ClientApp, type ClientConnection, + type ContentBlock, type RequestPermissionRequest, type SessionConfigOption, type SessionUpdate, @@ -58,7 +60,9 @@ import { boundedText, emitText, emitToolOutput, - projectToolResult, + projectRichToolResult, + projectContentBlock, + normalizeQueuedContent, promptText, summarizeToolContent, toolName, @@ -149,6 +153,10 @@ interface ActivePrompt { readonly context: PluginExecutorContext; readonly tools: Map; text: string; + updates: Promise; + updateError?: unknown; + readonly media: Map; + mediaBytes: number; } interface ToolSnapshot { @@ -158,6 +166,7 @@ interface ToolSnapshot { kind?: ToolCall['kind']; status?: ToolCall['status']; content: ToolCallContent[]; + contentTruncated?: boolean; rawInput?: unknown; rawOutput?: unknown; output: string; @@ -577,7 +586,15 @@ export class AcpExecutor implements PluginExecutorProvider { } if (session.active || session.configuring || session.awaitingAck) return failure(`${this.displayName} ACP Session is busy`, 'acp_busy', true); - const active: ActivePrompt = { context, tools: new Map(), text: '' }; + const mediaAbort = new AbortController(); + const active: ActivePrompt = { + context: { ...context, signal: AbortSignal.any([context.signal, mediaAbort.signal]) }, + tools: new Map(), + text: '', + updates: Promise.resolve(), + media: new Map(), + mediaBytes: 0, + }; session.active = active; try { await this.#ensureInitialized(session, context.signal); @@ -589,7 +606,12 @@ export class AcpExecutor implements PluginExecutorProvider { sessionId: session.acpSessionId!, prompt: [{ type: 'text', text: promptText(request) }], }); + // Keep observing this owner through media drain, even if prompt has already returned + // and the loss handler clears session.owner before the queued media settles. + const transportFailed = session.owner!.failed; const response = await this.#awaitPrompt(session, prompt, context.signal); + await Promise.race([active.updates, transportFailed]); + if (active.updateError) throw active.updateError; if (!response) return { status: 'cancelled', reason: 'timeout' }; // A settled prompt can be checkpointed even when it was cancelled or failed. // Timeouts and transport failures never reach this point, and the Runtime @@ -632,6 +654,9 @@ export class AcpExecutor implements PluginExecutorProvider { } return failure(safeErrorMessage(error), errorCode(error), retryable); } finally { + // Transport failure must also release media waits before draining received updates. + mediaAbort.abort(); + await active.updates; if (session.active === active) session.active = undefined; } } @@ -1134,7 +1159,70 @@ export class AcpExecutor implements PluginExecutorProvider { #configureClient(session: RetainedSession, app: ClientApp): void { app .onNotification(methods.client.session.update, ({ params }) => { - if (params.sessionId === session.acpSessionId) this.#acceptUpdate(session, params.update); + if (params.sessionId !== session.acpSessionId || session.lost || session.historyGap) return; + const active = session.active; + if ( + !active || + session.restoring || + params.update.sessionUpdate === 'config_option_update' + ) { + void this.#acceptUpdate(session, params.update); + return; + } + // Notifications without a live projection must not retain payloads behind slow media. + if ( + !['agent_message_chunk', 'agent_thought_chunk', 'tool_call', 'tool_call_update'].includes( + params.update.sessionUpdate, + ) || + (params.update.sessionUpdate === 'agent_thought_chunk' && + params.update.content.type !== 'text') + ) + return; + // Intern repeated snapshots before queuing: only unique media consumes the turn budget. + const reserve = (block: ContentBlock): ContentBlock => { + block = normalizeQueuedContent(block); + if (block.type === 'text') return block; + const serialized = JSON.stringify(block); + const size = Buffer.byteLength(serialized, 'utf8'); + const key = createHash('sha256').update(serialized).digest('hex'); + const existing = active.media.get(key); + if (existing) return existing; + if (active.media.size >= 64 || active.mediaBytes + size > 12 * 1024 * 1024) + return { type: 'text', text: '[Media omitted: response media limit reached]' }; + active.media.set(key, block); + active.mediaBytes += size; + return block; + }; + let update = params.update; + let contentTruncated = false; + if (update.sessionUpdate === 'agent_message_chunk') + update = { ...update, content: reserve(update.content) }; + else if ( + (update.sessionUpdate === 'tool_call' || update.sessionUpdate === 'tool_call_update') && + update.content + ) { + // Legacy text/diff results have their own size bound. Only mixed results + // consume the part budget, reserving one slot for an explicit omission. + contentTruncated = + update.content.length > SESSION_CONTENT_MAX_PARTS && + update.content.some((item) => item.type === 'content' && item.content.type !== 'text'); + const content = contentTruncated + ? update.content.slice(0, SESSION_CONTENT_MAX_PARTS - 1) + : update.content; + update = { + ...update, + content: content.map((item) => + item.type === 'content' ? { type: 'content', content: reserve(item.content) } : item, + ), + }; + } + active.updates = active.updates + // Admission belongs to notification receipt. A lost connection must not + // discard updates already accepted for this prompt or target a later prompt. + .then(() => this.#acceptPromptUpdate(active, update, contentTruncated)) + .catch((error) => { + active.updateError ??= error; + }); }) .onRequest(methods.client.fs.readTextFile, async ({ params }) => { this.#assertWorkspaceAccess(session, params.sessionId); @@ -1167,7 +1255,7 @@ export class AcpExecutor implements PluginExecutorProvider { return { outcome }; } - #acceptUpdate(session: RetainedSession, update: SessionUpdate): void { + async #acceptUpdate(session: RetainedSession, update: SessionUpdate): Promise { if (session.restoring) { if (session.replay) { session.replay.updates++; @@ -1187,17 +1275,35 @@ export class AcpExecutor implements PluginExecutorProvider { } const active = session.active; if (!active) return; + await this.#acceptPromptUpdate(active, update); + } + + async #acceptPromptUpdate( + active: ActivePrompt, + update: SessionUpdate, + contentTruncated = false, + ): Promise { if (update.sessionUpdate === 'agent_message_chunk' && update.content.type === 'text') { active.text += update.content.text; emitText(active.context, 'output_delta', update.content.text); return; } + if (update.sessionUpdate === 'agent_message_chunk') { + const text = sessionContentMarkdown( + await projectContentBlock(update.content, active.context), + ); + active.text += text; + emitText(active.context, 'output_delta', text); + return; + } if (update.sessionUpdate === 'agent_thought_chunk' && update.content.type === 'text') { emitText(active.context, 'thinking_delta', update.content.text); return; } - if (update.sessionUpdate === 'tool_call') this.#acceptTool(active, update); - if (update.sessionUpdate === 'tool_call_update') this.#acceptTool(active, update); + if (update.sessionUpdate === 'tool_call') + await this.#acceptTool(active, update, contentTruncated); + if (update.sessionUpdate === 'tool_call_update') + await this.#acceptTool(active, update, contentTruncated); } #assertWorkspaceAccess(session: RetainedSession, sessionId: string): void { @@ -1210,8 +1316,12 @@ export class AcpExecutor implements PluginExecutorProvider { } } - #acceptTool(active: ActivePrompt, update: ToolCall | ToolCallUpdate): void { - const snapshot = active.tools.get(update.toolCallId) ?? { + async #acceptTool( + active: ActivePrompt, + update: ToolCall | ToolCallUpdate, + contentTruncated = false, + ): Promise { + const snapshot: ToolSnapshot = active.tools.get(update.toolCallId) ?? { id: update.toolCallId, title: 'External tool', content: [], @@ -1224,7 +1334,10 @@ export class AcpExecutor implements PluginExecutorProvider { if (update.name != null) snapshot.name = update.name; if (update.kind != null) snapshot.kind = update.kind; if (update.status != null) snapshot.status = update.status; - if (update.content != null) snapshot.content = [...update.content]; + if (update.content != null) { + snapshot.content = [...update.content]; + snapshot.contentTruncated = contentTruncated; + } if (update.rawInput !== undefined) snapshot.rawInput = update.rawInput; if (update.rawOutput !== undefined) snapshot.rawOutput = update.rawOutput; active.tools.set(snapshot.id, snapshot); @@ -1258,7 +1371,12 @@ export class AcpExecutor implements PluginExecutorProvider { active.context.emit({ type: 'tool_result', toolCallId: snapshot.id, - content: projectToolResult(snapshot.content, snapshot.rawOutput), + content: await projectRichToolResult( + snapshot.content, + snapshot.rawOutput, + active.context, + snapshot.contentTruncated, + ), ...(snapshot.status === 'failed' ? { isError: true } : {}), }); snapshot.terminal = true; diff --git a/packages/cli/src/pi-transcript-format.ts b/packages/cli/src/pi-transcript-format.ts index 420b6a55b86..abd963f7cfb 100644 --- a/packages/cli/src/pi-transcript-format.ts +++ b/packages/cli/src/pi-transcript-format.ts @@ -48,6 +48,8 @@ export function fitLine(line: string, width: number): string { export function formatToolResultContent(content: ToolResultContent): string { switch (content.kind) { + case 'content': + return content.parts.map(formatToolResultContent).join('\n\n'); case 'text': return content.text; case 'json': diff --git a/packages/core/package.json b/packages/core/package.json index 83b3ba5a835..81cf7e0d488 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -33,6 +33,7 @@ "./provider-retry-countdown": "./dist/provider-retry-countdown.js", "./message-queue-order": "./dist/message-queue-order.js", "./interaction": "./dist/interaction.js", + "./session-media": "./dist/session-media.js", "./session": "./dist/session.js", "./turn-origin": "./dist/turn-origin.js", "./session-reference": "./dist/session-reference.js", diff --git a/packages/core/src/__tests__/session-media.test.ts b/packages/core/src/__tests__/session-media.test.ts new file mode 100644 index 00000000000..2119cd7e682 --- /dev/null +++ b/packages/core/src/__tests__/session-media.test.ts @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + isSessionMediaInput, + isSessionContentPart, + sessionContentMarkdown, + SESSION_MEDIA_MAX_BASE64, +} from '../session-media.js'; + +test('media transport bounds binary input and only publishes durable image references', () => { + assert.ok(isSessionMediaInput({ kind: 'inline', mimeType: 'image/png', base64: 'AAAA' })); + for (const base64 of ['not base64', 'AA', 'AAAA=', 'A'.repeat(SESSION_MEDIA_MAX_BASE64 + 4)]) + assert.equal(isSessionMediaInput({ kind: 'inline', mimeType: 'image/png', base64 }), false); + assert.equal( + isSessionMediaInput({ kind: 'source', source: '/tmp/image.svg', extra: true }), + false, + ); + assert.equal(isSessionMediaInput({ kind: 'source', source: '/tmp/\0image.svg' }), false); + const part = { + kind: 'image' as const, + mimeType: 'image/svg+xml', + ref: { kind: 'session_file' as const, sessionId: 'session', relativePath: 'image-1' }, + }; + assert.ok(isSessionContentPart(part)); + assert.equal(sessionContentMarkdown(part), '\n\n![](maka://runtime/attachments/image-1)\n\n'); + assert.equal( + isSessionContentPart({ ...part, ref: { ...part.ref, relativePath: '../other' } }), + false, + ); +}); + +test('ordered media survives canonical tool-result validation without allowing nested payloads', async () => { + const { decodeCanonicalToolResultContent } = await import('../tool-result-record-schema.js'); + const content = { + kind: 'content', + parts: [ + { kind: 'text', text: 'before' }, + { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session', relativePath: 'image-1' }, + }, + ], + }; + assert.deepEqual(decodeCanonicalToolResultContent(content), content); + assert.throws(() => decodeCanonicalToolResultContent({ kind: 'content', parts: [content] })); + assert.throws(() => + decodeCanonicalToolResultContent({ + kind: 'content', + parts: Array.from({ length: 65 }, () => ({ kind: 'text', text: 'a' })), + }), + ); +}); diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index eec5e787621..4acb7f644ce 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -946,6 +946,7 @@ type ShellRunToolResultContent = )); export type ToolResultContent = + | { kind: 'content'; parts: import('./session-media.js').SessionContentPart[] } | { kind: 'text'; text: string; diff --git a/packages/core/src/image-delivery.ts b/packages/core/src/image-delivery.ts index c89a0944176..54ef4b45253 100644 --- a/packages/core/src/image-delivery.ts +++ b/packages/core/src/image-delivery.ts @@ -18,6 +18,9 @@ */ /** Client-safe delivery contract. No filesystem or networking capabilities live here. */ +/** Original-file admission and renderer preview budgets are separate policies. */ +export const IMAGE_ARCHIVE_MAX_BYTES = 8 * 1024 * 1024; + export const IMAGE_DELIVERY_IDENTITY_MAX_LENGTH = 512; export const IMAGE_DELIVERY_SOURCE_MAX_LENGTH = 4096; /** JavaScript string length, shared by stream capture and Markdown consumers. */ @@ -106,6 +109,8 @@ export type ResolveImageDelivery = ( ) => Promise; /** Stored with its Artifact, so session copy/export carries both provenance and bytes. */ export interface ImageDeliveryMetadata { + /** Executor publication owner, retained across a crash before transcript publication. */ + readonly publicationRunId?: string; readonly messageId: string; readonly source: string; readonly status: 'ready'; @@ -123,7 +128,14 @@ export function isImageDeliveryMetadata(value: unknown): value is ImageDeliveryM if (!value || typeof value !== 'object' || Array.isArray(value)) return false; const v = value as Record; return ( - Object.keys(v).every((k) => ['messageId', 'source', 'status', 'contentSha256'].includes(k)) && + Object.keys(v).every((k) => + ['messageId', 'source', 'status', 'contentSha256', 'publicationRunId'].includes(k), + ) && + (v.publicationRunId === undefined || + (typeof v.publicationRunId === 'string' && + v.publicationRunId.length > 0 && + v.publicationRunId.length <= 512 && + !/[\x00-\x1f\x7f]/.test(v.publicationRunId))) && typeof v.messageId === 'string' && v.messageId.length > 0 && v.messageId.length <= IMAGE_DELIVERY_IDENTITY_MAX_LENGTH && diff --git a/packages/core/src/session-media.ts b/packages/core/src/session-media.ts new file mode 100644 index 00000000000..acaf5da52b7 --- /dev/null +++ b/packages/core/src/session-media.ts @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IMAGE_ARCHIVE_MAX_BYTES } from './image-delivery.js'; +import type { StorageRef } from './events.js'; +import { isCanonicalArtifactEntityId } from './artifacts.js'; +import { formatAttachmentResourceRef } from './attachments.js'; + +/** Transport-neutral input. Only the Host may turn a source into a saved reference. */ +export type SessionMediaInput = + | { kind: 'inline'; mimeType: string; base64: string } + | { kind: 'source'; source: string }; + +export type SessionContentPart = + | { kind: 'text'; text: string } + | { kind: 'image'; mimeType: string; ref: Extract } + | { kind: 'file_diff'; paths: string[]; diff: string }; + +export const SESSION_CONTENT_MAX_PARTS = 64; +export const SESSION_MEDIA_MAX_BYTES = IMAGE_ARCHIVE_MAX_BYTES; +export const SESSION_MEDIA_MAX_BASE64 = 4 * Math.ceil(SESSION_MEDIA_MAX_BYTES / 3); + +export function isSessionMediaInput(value: unknown): value is SessionMediaInput { + if (!value || typeof value !== 'object') return false; + const v = value as Record; + if (v.kind === 'source') + return ( + Object.keys(v).length === 2 && + typeof v.source === 'string' && + v.source.length > 0 && + v.source.length <= 4096 && + !/[\x00-\x1f\x7f]/.test(v.source) + ); + return ( + v.kind === 'inline' && + Object.keys(v).length === 3 && + typeof v.mimeType === 'string' && + /^image\/[a-z0-9.+-]{1,64}$/.test(v.mimeType) && + typeof v.base64 === 'string' && + v.base64.length > 0 && + v.base64.length <= SESSION_MEDIA_MAX_BASE64 && + v.base64.length % 4 === 0 && + /^[A-Za-z0-9+/]*={0,2}$/.test(v.base64) + ); +} + +export function isSessionContentPart(value: unknown): value is SessionContentPart { + if (!value || typeof value !== 'object') return false; + const v = value as Record; + if (v.kind === 'text') + return Object.keys(v).length === 2 && typeof v.text === 'string' && v.text.length <= 8192; + if (v.kind === 'file_diff') + return ( + Object.keys(v).length === 3 && + Array.isArray(v.paths) && + v.paths.length > 0 && + v.paths.length <= 64 && + v.paths.every((p) => typeof p === 'string' && p.length <= 8192) && + typeof v.diff === 'string' && + v.diff.length <= 1024 * 1024 + ); + if ( + v.kind !== 'image' || + Object.keys(v).length !== 3 || + typeof v.mimeType !== 'string' || + !/^image\/[a-z0-9.+-]{1,64}$/.test(v.mimeType) || + !v.ref || + typeof v.ref !== 'object' + ) + return false; + const ref = v.ref as Record; + return ( + Object.keys(ref).length === 3 && + ref.kind === 'session_file' && + typeof ref.sessionId === 'string' && + ref.sessionId.length > 0 && + ref.sessionId.length <= 512 && + isCanonicalArtifactEntityId(ref.relativePath) + ); +} + +/** Compatibility projection: ordered media uses the existing durable Markdown transcript. */ +export function sessionContentMarkdown( + part: Exclude, +): string { + if (part.kind === 'text') return part.text; + const source = formatAttachmentResourceRef(part.ref); + return source ? `\n\n![](${source})\n\n` : ''; +} diff --git a/packages/core/src/tool-result-record-schema.ts b/packages/core/src/tool-result-record-schema.ts index 6f46fecc16c..7536ee09d46 100644 --- a/packages/core/src/tool-result-record-schema.ts +++ b/packages/core/src/tool-result-record-schema.ts @@ -17,6 +17,7 @@ * under the License. */ +import { isSessionContentPart, SESSION_CONTENT_MAX_PARTS } from './session-media.js'; import { decodeCanonicalShellToolResultContent, isSandboxDenialSignal, @@ -260,6 +261,14 @@ function isNonShellToolResultContent(value: unknown): value is ToolResultContent value.reason === 'stale_tool_result_pruned_before_compact' || value.reason === 'active_current_turn_tool_result_pruned_before_next_step') ); + case 'content': + return ( + Object.keys(value).length === 2 && + Array.isArray(value.parts) && + value.parts.length > 0 && + value.parts.length <= SESSION_CONTENT_MAX_PARTS && + value.parts.every(isSessionContentPart) + ); case 'image': return ( hasExactShape(value, IMAGE_SHAPE) && diff --git a/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts index f7b7c325dab..ac967662667 100644 --- a/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts +++ b/packages/runtime-host/src/__tests__/chat-image-delivery.test.ts @@ -116,18 +116,24 @@ async function fixture( const messages = new Map(); const localReader = createImageFileReader(); let present = true; - const service = new ChatImageDeliveryService({ + const ports: ChatImageDeliveryPorts = { artifacts: { + putImageDeliveryJob: authority.store.putImageDeliveryJob, + readImageDeliveryJob: authority.store.readImageDeliveryJob, + listImageDeliveryJobs: authority.store.listImageDeliveryJobs, + deleteImageDeliveryJob: authority.store.deleteImageDeliveryJob, create: async (input) => { beforeCreate?.(input); return authority.store.create(input); }, findImageDelivery: authority.store.findImageDelivery, + getInSession: authority.store.getInSession, setImageDeliveryAttempt: authority.store.setImageDeliveryAttempt, }, admission: new SessionAdmissionGate(), limits, sourceReadTimeoutMs: 1000, + retryDelayMs: 1, canLoadRemote: async () => remoteAllowed, download: download ?? downloadChatImage, isPresent: async () => present, @@ -147,9 +153,11 @@ async function fixture( }, persistenceFailed: (error) => errors.push(error), presentationFailed: (error) => presentationErrors.push(error), - }); + }; + const service = new ChatImageDeliveryService(ports); return { root, + ports, owner, authority, service, @@ -183,8 +191,8 @@ test('Host drain releases both capture slots even when a source reader ignores c }, async () => { const f = await fixture(undefined, () => new Promise(() => {})); try { - observe(f.service, '/tmp/blocked-1.png'); - observe(f.service, '/tmp/blocked-2.png', 'session-1', 'message-2'); + await observe(f.service, '/tmp/blocked-1.png'); + await observe(f.service, '/tmp/blocked-2.png', 'session-1', 'message-2'); while (f.reads < 2) await new Promise((resolve) => setImmediate(resolve)); await f.service.close(); assert.equal(f.leases, 0); @@ -193,7 +201,7 @@ test('Host drain releases both capture slots even when a source reader ignores c await f.close(); } }); -test('a stalled local source expires within its read budget and can be retried', { +test('a stalled local source automatically retries within a bounded attempt budget', { timeout: 15000, }, async () => { let reads = 0; @@ -202,12 +210,10 @@ test('a stalled local source expires within its read budget and can be retried', return checkedChatImage(PNG); }); try { - observe(f.service, REQUEST.source); + await observe(f.service, REQUEST.source); await f.service.waitForIdle(); - assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: 'read_failed' }); + assert.equal(reads, 2); assert.equal(f.leases, 0); - assert.equal((await f.service.resolve({ ...REQUEST, retry: true })).status, 'pending'); - await f.service.waitForIdle(); assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); assert.deepEqual(f.errors, []); } finally { @@ -220,7 +226,7 @@ function observe( sessionId = REQUEST.sessionId, messageId = REQUEST.messageId, ) { - service.observe(sessionId, { + return service.observe(sessionId, { id: 'text-event', type: 'text_complete', turnId: REQUEST.turnId, @@ -256,7 +262,7 @@ test('automatically saves a local delivery without any UI request; deletion and try { const source = join(f.root, 'original.png'); await writeFile(source, PNG); - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); assert.equal(f.reads, 1); assert.equal(f.leases, 0); @@ -302,7 +308,7 @@ test('literal percent filenames retain precedence and file URLs are decoded only if (name.includes('%20')) await writeFile(path.replace('%20', ' '), 'wrong source'); for (const source of [path, pathToFileURL(path).href]) { paths.length = 0; - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready', source); assert.deepEqual(paths, [path]); @@ -332,16 +338,9 @@ test('remote delivery is downloaded once, remains replayable after the origin se ); try { f.messages.set(REQUEST.messageId, `![Screenshot](${source})`); - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); - assert.equal(requests, 0); - assert.deepEqual(await f.service.resolve({ ...REQUEST, source }), { - status: 'requires_confirmation', - }); - assert.equal( - (await f.service.resolve({ ...REQUEST, source, loadRemote: true })).status, - 'pending', - ); + assert.equal(requests, 1); await f.service.waitForIdle(); assert.deepEqual(f.errors, []); assert.equal((await f.service.resolve({ ...REQUEST, source })).status, 'ready'); @@ -370,7 +369,7 @@ test('transient failure is persisted and is retried only on an explicit request' const f = await fixture(); try { const source = join(f.root, 'later.png'); - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); assert.deepEqual(await f.service.resolve({ ...REQUEST, source }), { status: 'failed', @@ -444,7 +443,7 @@ test('ready image retries retain the only saved copy after its source is deleted const source = join(f.root, 'temporary.png'); try { await writeFile(source, PNG); - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); const identity = { ...REQUEST, source }; const ready = await f.service.resolve(identity); @@ -496,7 +495,7 @@ test('Worker read errors retain their image delivery reasons in persisted failur reader({ path, cwd: process.cwd(), abortSignal }), ); try { - observe(f.service, REQUEST.source); + await observe(f.service, REQUEST.source); await f.service.waitForIdle(); assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: expected }); assert.deepEqual(f.errors, []); @@ -517,7 +516,7 @@ test('pending capture survives persistence failure and is removed only after a d }, ); try { - observe(f.service, REQUEST.source); + await observe(f.service, REQUEST.source); await f.service.waitForIdle(); const pending = await f.authority.store.listPage(REQUEST.sessionId, { offset: 0, limit: 10 }); assert.equal(pending.total, 0, 'pending capture is not an artifact'); @@ -548,8 +547,8 @@ test('quota admission is atomic under concurrent jobs and identical content shar try { const source = join(f.root, 'same.png'); await writeFile(source, PNG); - observe(f.service, source); - observe(f.service, source, 'session-2'); + await observe(f.service, source); + await observe(f.service, source, 'session-2'); await f.service.waitForIdle(); const a = await f.service.resolve({ ...REQUEST, source }); const b = await f.service.resolve({ ...REQUEST, sessionId: 'session-2', source }); @@ -564,7 +563,7 @@ test('quota admission is atomic under concurrent jobs and identical content shar ); const different = join(f.root, 'different.png'); await writeFile(different, Buffer.concat([PNG, Buffer.from('different')])); - observe(f.service, different, 'session-1', 'message-2'); + await observe(f.service, different, 'session-1', 'message-2'); await f.service.waitForIdle(); assert.deepEqual( await f.service.resolve({ ...REQUEST, source: different, messageId: 'message-2' }), @@ -582,7 +581,7 @@ test('copying a conversation carries delivery provenance and saved bytes; it doe try { const source = join(f.root, 'original.png'); await writeFile(source, PNG); - observe(f.service, source); + await observe(f.service, source); await f.service.waitForIdle(); await rm(source); await f.authority.store.copyConversationArtifacts({ @@ -633,11 +632,11 @@ test('source readers reject unsafe MIME and redirects to private networks', asyn test('HTTP download enforces header/stream limits, redirect budget and cancellation', async (t) => { const server = createServer((req, res) => { if (req.url === '/length') { - res.writeHead(200, { 'content-length': 2 * 1024 * 1024 + 1 }); + res.writeHead(200, { 'content-length': 8 * 1024 * 1024 + 1 }); res.end(); } else if (req.url === '/stream') { res.writeHead(200); - res.write(Buffer.alloc(2 * 1024 * 1024 + 1)); + res.write(Buffer.alloc(8 * 1024 * 1024 + 1)); res.end(); } else if (req.url === '/redirect') { res.writeHead(302, { location: '/redirect' }); @@ -722,8 +721,6 @@ test('remote capture requires client display authority and application network a const identity = { ...REQUEST, source }; f.messages.set(REQUEST.messageId, `![](${source})`); try { - observe(f.service, source); - await f.service.waitForIdle(); assert.equal(downloads, 0); assert.deepEqual(await f.service.resolve(identity), { status: 'requires_confirmation' }); f.allowRemote(false); @@ -764,7 +761,7 @@ test('remote capture requires client display authority and application network a } }); -test('failed remote downloads issue one request and require a client retry', async () => { +test('remote downloads retry three times, then require an explicit authorized retry', async () => { let downloads = 0; const f = await fixture(undefined, undefined, undefined, async () => { downloads++; @@ -776,14 +773,14 @@ test('failed remote downloads issue one request and require a client retry', asy try { await f.service.resolve({ ...identity, loadRemote: true }); await f.service.waitForIdle(); - assert.equal(downloads, 1); + assert.equal(downloads, 3); assert.deepEqual(await f.service.resolve({ ...identity, retry: true }), { status: 'requires_confirmation', }); - assert.equal(downloads, 1); + assert.equal(downloads, 3); await f.service.resolve({ ...identity, retry: true, loadRemote: true }); await f.service.waitForIdle(); - assert.equal(downloads, 2); + assert.equal(downloads, 6); } finally { await f.close(); } @@ -792,7 +789,7 @@ test('failed remote downloads issue one request and require a client retry', asy test('a Markdown parser failure stays in presentation and subsequent images still capture', async () => { const f = await fixture(undefined, async () => checkedChatImage(PNG)); try { - f.service.observe(REQUEST.sessionId, { + await f.service.observe(REQUEST.sessionId, { type: 'text_complete', id: 'deep-text', ts: 1, @@ -803,7 +800,7 @@ test('a Markdown parser failure stays in presentation and subsequent images stil assert.equal(f.presentationErrors.length, 1); assert.ok(f.presentationErrors[0] instanceof RangeError); assert.deepEqual(f.errors, []); - observe(f.service, REQUEST.source); + await observe(f.service, REQUEST.source); await f.service.waitForIdle(); assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); } finally { @@ -885,7 +882,7 @@ test('local and downloaded invalid image bytes retain identical failure reasons' try { for (const [bytes, reason] of [ [PNG.subarray(0, 8), 'unsupported_mime'], - [new Uint8Array(2 * 1024 * 1024 + 1), 'too_large'], + [new Uint8Array(8 * 1024 * 1024 + 1), 'too_large'], ] as const) { assert.throws( () => checkedChatImage(bytes), @@ -915,11 +912,11 @@ test('a queued retry reports pending instead of replaying the previous failure', return checkedChatImage(PNG); }); try { - observe(f.service, REQUEST.source); + await observe(f.service, REQUEST.source); await f.service.waitForIdle(); assert.deepEqual(await f.service.resolve(REQUEST), { status: 'failed', reason: 'not_found' }); - observe(f.service, '/tmp/blocker-one.png'); - observe(f.service, '/tmp/blocker-two.png'); + await observe(f.service, '/tmp/blocker-one.png'); + await observe(f.service, '/tmp/blocker-two.png'); while (f.reads < 3) await new Promise((resolve) => setImmediate(resolve)); fail = false; assert.deepEqual(await f.service.resolve({ ...REQUEST, retry: true }), { status: 'pending' }); @@ -933,3 +930,542 @@ test('a queued retry reports pending instead of replaying the previous failure', await f.close(); } }); + +test('executor inline media and Markdown use the same archive, deduplicate and replay after source deletion', async () => { + let writes = 0; + const f = await fixture(undefined, undefined, () => { + writes++; + }); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + try { + const input = { + kind: 'inline' as const, + mimeType: 'image/svg+xml', + base64: PNG.toString('base64'), + }; + const [a, b] = await Promise.all([ + f.service.publish(owner, input), + f.service.publish(owner, input), + ]); + assert.deepEqual(a, b); + assert.equal(a.kind, 'image'); + assert.equal(writes, 1); + if (a.kind !== 'image') throw new Error('Image not published'); + assert.equal(a.mimeType, 'image/png'); // Bytes, not the external MIME claim, are authoritative. + const read = await f.authority.store.readBinaryInSession(owner.sessionId, a.ref.relativePath); + assert.ok(read.ok); + if (read.ok) assert.equal(read.base64, PNG.toString('base64')); + const file = join(f.root, 'drawing.png'); + await writeFile(file, PNG); + const saved = await f.service.publish(owner, { kind: 'source', source: file }); + assert.equal(saved.kind, 'image'); + if (saved.kind !== 'image') throw new Error('Image not published'); + const imageRead = await f.authority.store.readBinaryInSession( + owner.sessionId, + saved.ref.relativePath, + ); + assert.ok(imageRead.ok); + if (imageRead.ok) assert.equal(imageRead.mimeType, 'image/png'); + await rm(file); + const replay = await f.authority.store.readBinaryInSession( + owner.sessionId, + saved.ref.relativePath, + ); + assert.ok(replay.ok); + if (replay.ok) assert.equal(replay.base64, PNG.toString('base64')); + assert.equal(f.reads, 1); + assert.equal(f.leases, 0); + } finally { + await f.close(); + } +}); + +test('executor publication keeps invalid bytes, quota and network failures out of image references', async () => { + const f = await fixture({ sessionBytes: 10, workspaceBytes: 10 }); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + try { + const invalid = await f.service.publish(owner, { + kind: 'inline', + mimeType: 'image/png', + base64: Buffer.from('not an image').toString('base64'), + }); + assert.equal(invalid.kind, 'text'); + const quota = await f.service.publish(owner, { + kind: 'inline', + mimeType: 'image/png', + base64: PNG.toString('base64'), + }); + assert.deepEqual(quota, { kind: 'text', text: '[Media unavailable: quota_exceeded]' }); + f.allowRemote(false); + assert.deepEqual( + await f.service.publish(owner, { kind: 'source', source: 'https://example.com/image.svg' }), + { kind: 'text', text: '[Media unavailable: not_allowed]' }, + ); + assert.equal(f.leases, 0); + } finally { + await f.close(); + } +}); + +test('separate source publications archive changed bytes in the same turn and keep both references readable', async () => { + const f = await fixture(); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + const file = join(f.root, 'plot.png'); + const updated = Buffer.concat([PNG, Buffer.from('second version')]); + try { + await writeFile(file, PNG); + const first = await f.service.publish(owner, { kind: 'source', source: file }); + await writeFile(file, updated); + const second = await f.service.publish(owner, { kind: 'source', source: file }); + assert.equal(first.kind, 'image'); + assert.equal(second.kind, 'image'); + if (first.kind !== 'image' || second.kind !== 'image') throw new Error('Image not published'); + assert.notEqual(first.ref.relativePath, second.ref.relativePath); + assert.equal(f.reads, 2); + await rm(file); + for (const [part, bytes] of [ + [first, PNG], + [second, updated], + ] as const) { + const replay = await f.authority.store.readBinaryInSession( + owner.sessionId, + part.ref.relativePath, + ); + assert.ok(replay.ok); + if (replay.ok) assert.equal(replay.base64, bytes.toString('base64')); + } + assert.equal((await f.service.publish(owner, { kind: 'source', source: file })).kind, 'text'); + assert.equal(f.leases, 0); + assert.deepEqual(f.errors, []); + } finally { + await f.close(); + } +}); + +test('concurrent source publications return their own archive despite reversed capture order', { + timeout: 5000, +}, async () => { + let release!: () => void; + let started!: () => void; + const firstStarted = new Promise((resolve) => { + started = resolve; + }); + const blocked = new Promise((resolve) => { + release = resolve; + }); + const updated = Buffer.concat([PNG, Buffer.from('second version')]); + let reads = 0; + const f = await fixture(undefined, async () => { + if (++reads === 1) { + started(); + await blocked; + return checkedChatImage(PNG); + } + return checkedChatImage(updated); + }); + const owner = { sessionId: 'session-1', turnId: 'turn-1', messageId: 'message-1' }; + try { + const firstPending = f.service.publish(owner, { kind: 'source', source: '/plot.png' }); + await firstStarted; + const second = await f.service.publish(owner, { kind: 'source', source: '/plot.png' }); + release(); + const first = await firstPending; + assert.equal(first.kind, 'image'); + assert.equal(second.kind, 'image'); + if (first.kind !== 'image' || second.kind !== 'image') throw new Error('Image not published'); + assert.notEqual(first.ref.relativePath, second.ref.relativePath); + for (const [part, bytes] of [ + [first, PNG], + [second, updated], + ] as const) { + const saved = await f.authority.store.readBinaryInSession( + owner.sessionId, + part.ref.relativePath, + ); + assert.ok(saved.ok); + if (saved.ok) assert.equal(saved.base64, bytes.toString('base64')); + } + assert.equal(reads, 2); + assert.equal(f.leases, 0); + assert.deepEqual(f.errors, []); + } finally { + release(); + await f.close(); + } +}); + +test('startup finishes a durably staged image after the source disappears and without a mounted client', async () => { + let failCommit = true; + const f = await fixture( + undefined, + async () => checkedChatImage(PNG), + () => { + if (failCommit) throw new Error('simulated process interruption before Artifact commit'); + }, + ); + let resumed: ChatImageDeliveryService | undefined; + try { + await observe(f.service, REQUEST.source); + await f.service.waitForIdle(); + assert.equal(f.errors.length, 1); + const pending = await f.authority.store.listImageDeliveryJobs(Date.now()); + assert.equal(pending.length, 1); + const staged = await f.authority.store.readImageDeliveryJob(pending[0]!.id); + assert.deepEqual(Buffer.from(staged!.bytes!), PNG); + await f.service.close(); + failCommit = false; + resumed = new ChatImageDeliveryService({ + ...f.ports, + readLocalImage: async () => { + assert.fail('recovery must use received bytes, not the deleted source'); + }, + }); + await resumed.start(); + await resumed.waitForIdle(); + const ready = await resumed.resolve(REQUEST); + assert.equal(ready.status, 'ready'); + assert.deepEqual(await f.authority.store.listImageDeliveryJobs(Date.now()), []); + if (ready.status === 'ready') { + const bytes = await f.authority.store.readBinaryInSession( + REQUEST.sessionId, + ready.artifactId, + ); + assert.ok(bytes.ok); + if (bytes.ok) assert.deepEqual(Buffer.from(bytes.base64, 'base64'), PNG); + } + } finally { + await resumed?.close(); + await f.close(); + } +}); + +test('startup resumes a queued authorized source without a viewport request', async () => { + const f = await fixture(undefined, async () => checkedChatImage(PNG)); + try { + await f.authority.store.putImageDeliveryJob({ + job: { + ...REQUEST, + id: 'recovered-image', + attempts: 0, + nextAttemptAt: 0, + loadRemote: false, + }, + }); + await f.service.start(); + await f.service.waitForIdle(); + assert.equal(f.reads, 1); + assert.equal((await f.service.resolve(REQUEST)).status, 'ready'); + } finally { + await f.close(); + } +}); + +test('Host captures allowed remote images without a viewport and does not fetch redacted or disallowed sources', async () => { + let downloads = 0; + const f = await fixture(undefined, undefined, undefined, async () => { + downloads++; + return checkedChatImage(PNG); + }); + try { + await observe(f.service, 'https://example.invalid/image.png'); + await f.service.waitForIdle(); + assert.equal(downloads, 1); + assert.equal( + (await f.service.resolve({ ...REQUEST, source: 'https://example.invalid/image.png' })).status, + 'ready', + ); + await observe(f.service, 'https://example.invalid/image.png?api_key=secret-value'); + f.allowRemote(false); + await observe(f.service, 'https://example.invalid/second.png'); + await f.service.waitForIdle(); + assert.equal(downloads, 1); + } finally { + await f.close(); + } +}); + +test('a restarted Host never retries a terminal source failure or resurrects a removed Session', async () => { + const f = await fixture(undefined, async () => checkedChatImage(PNG)); + try { + await f.authority.store.putImageDeliveryJob({ + job: { + ...REQUEST, + id: 'failed-image', + attempts: 3, + nextAttemptAt: 0, + loadRemote: false, + failure: 'read_failed', + }, + }); + await f.service.start(); + await f.service.waitForIdle(); + assert.equal(f.reads, 0); + await f.authority.store.putImageDeliveryJob({ + job: { + ...REQUEST, + id: 'deleted-session-image', + attempts: 0, + nextAttemptAt: 0, + loadRemote: false, + }, + }); + f.removeSession(); + await f.service.close(); + const resumed = new ChatImageDeliveryService(f.ports); + try { + await resumed.start(); + await resumed.waitForIdle(); + assert.equal(f.reads, 0); + assert.equal( + await f.authority.store.readImageDeliveryJob('deleted-session-image'), + undefined, + ); + } finally { + await resumed.close(); + } + } finally { + await f.close(); + } +}); + +test('valid originals above the inline preview budget are archived byte-for-byte', async () => { + const f = await fixture(); + const original = Buffer.concat([PNG, Buffer.alloc(2 * 1024 * 1024)]); + const path = join(f.root, 'large-original.png'); + try { + await writeFile(path, original); + await observe(f.service, path); + await f.service.waitForIdle(); + const saved = await f.service.resolve({ ...REQUEST, source: path }); + assert.equal(saved.status, 'ready'); + if (saved.status !== 'ready') return; + await rm(path); + const bytes = await f.authority.store.readBinaryInSession(REQUEST.sessionId, saved.artifactId, { + maxBytes: original.length, + }); + assert.ok(bytes.ok); + if (bytes.ok) assert.deepEqual(Buffer.from(bytes.base64, 'base64'), original); + } finally { + await f.close(); + } +}); + +test('inline publication accepts the exact archive byte limit with Base64 padding', async () => { + const f = await fixture(); + try { + const bytes = Buffer.alloc(8 * 1024 * 1024); + PNG.copy(bytes); + const part = await f.service.publish(REQUEST, { + kind: 'inline', + mimeType: 'image/png', + base64: bytes.toString('base64'), + }); + assert.equal(part.kind, 'image'); + const tooLarge = await f.service.publish(REQUEST, { + kind: 'inline', + mimeType: 'image/png', + base64: Buffer.alloc(bytes.length + 1).toString('base64'), + }); + assert.equal(tooLarge.kind, 'text'); + } finally { + await f.close(); + } +}); + +for (const crashPoint of ['spooled', 'archived'] as const) { + test(`startup reconnects ${crashPoint} executor images to the canonical transcript exactly once`, async () => { + const { createSqliteRuntimeStore } = await import('@maka/storage/sqlite-runtime-store'); + const { seedInvocation } = await import('@maka/runtime/test-only/invocation-fixture'); + const { projectRuntimeEventsToStoredMessages } = await import( + '@maka/runtime/runtime-event-read-model' + ); + const { recoverChatImagePublications } = await import('../server/chat-image-recovery.js'); + const f = await fixture(undefined, undefined, () => { + if (crashPoint === 'spooled') throw new Error('interrupted before Artifact commit'); + }); + const events = createSqliteRuntimeStore(join(f.root, 'recovery-runtime.sqlite')); + let recovery: ChatImageDeliveryService | undefined; + try { + await seedInvocation(events, { ...REQUEST, runId: 'run-1' }); + await f.service.publish( + { ...REQUEST, runId: 'run-1' }, + { kind: 'inline', mimeType: 'image/png', base64: PNG.toString('base64') }, + ); + await f.service.close(); + recovery = new ChatImageDeliveryService({ + ...f.ports, + artifacts: { ...f.ports.artifacts, create: f.authority.store.create }, + }); + const input = { + sessionIds: [REQUEST.sessionId], + events, + artifacts: f.authority.store, + delivery: recovery, + }; + await recoverChatImagePublications(input); + await recoverChatImagePublications(input); + const invocation = await events.readRunInvocation(REQUEST.sessionId, 'run-1'); + assert.ok(invocation); + const saved = await events.readImmutableRuntimeEvents(REQUEST.sessionId, 'run-1'); + const recovered = saved.filter((event) => event.refs?.artifactId); + assert.equal(recovered.length, 1); + assert.equal(recovered[0]?.author, 'host'); + assert.equal(recovered[0]?.modelVisibility, 'hidden'); + const transcript = projectRuntimeEventsToStoredMessages(saved, { invocations: [invocation] }); + const message = transcript.messages.find((message) => message.type === 'assistant'); + assert.ok(message?.type === 'assistant'); + assert.match(message.text, /Recovered image.*interruption/); + assert.match(message.text, /maka:\/\/runtime\/attachments\//); + const image = await f.authority.store.readDurableAttachmentBinary({ + sessionId: REQUEST.sessionId, + artifactId: recovered[0]!.refs!.artifactId!, + }); + assert.ok(image.ok); + assert.deepEqual(Buffer.from(image.base64, 'base64'), PNG); + await events.appendRuntimeEvent(REQUEST.sessionId, 'run-1', { + id: 'terminal', + sessionId: REQUEST.sessionId, + turnId: REQUEST.turnId, + runId: 'run-1', + invocationId: invocation.invocationId, + ts: Date.now(), + partial: false, + role: 'system', + author: 'host', + status: 'failed', + }); + await recoverChatImagePublications(input); + assert.equal( + (await events.readImmutableRuntimeEvents(REQUEST.sessionId, 'run-1')).length, + saved.length + 1, + ); + } finally { + await recovery?.close(); + events.close(); + await f.close(); + } + }); +} + +for (const placement of ['body', 'tool'] as const) { + test(`startup preserves a committed ${placement} image and registers missing body reception work`, async () => { + const { createSqliteRuntimeStore } = await import('@maka/storage/sqlite-runtime-store'); + const { seedInvocation } = await import('@maka/runtime/test-only/invocation-fixture'); + const { recoverChatImagePublications } = await import('../server/chat-image-recovery.js'); + const f = await fixture(); + const events = createSqliteRuntimeStore(join(f.root, 'recovery-runtime.sqlite')); + try { + await seedInvocation(events, { ...REQUEST, runId: 'run-1' }); + const part = await f.service.publish( + { ...REQUEST, runId: 'run-1' }, + { kind: 'inline', mimeType: 'image/png', base64: PNG.toString('base64') }, + ); + assert.equal(part.kind, 'image'); + assert.ok(part.kind === 'image'); + const identity = { + sessionId: REQUEST.sessionId, + turnId: REQUEST.turnId, + runId: 'run-1', + invocationId: 'run-1', + partial: false, + ts: 2, + }; + if (placement === 'tool') + await events.appendRuntimeEvent(REQUEST.sessionId, 'run-1', { + ...identity, + id: 'call', + role: 'model', + author: 'agent', + content: { kind: 'function_call', id: 'draw', name: 'Draw', args: {} }, + }); + await events.appendRuntimeEvent(REQUEST.sessionId, 'run-1', { + ...identity, + id: 'published', + role: placement === 'body' ? 'model' : 'tool', + author: placement === 'body' ? 'agent' : 'tool', + content: + placement === 'body' + ? { + kind: 'text', + text: `![saved](maka://runtime/attachments/${part.ref.relativePath})`, + } + : { + kind: 'function_response', + id: 'draw', + name: 'Draw', + result: { kind: 'content', parts: [part] }, + }, + }); + const path = join(f.root, 'unobserved.png'); + await writeFile(path, PNG); + await events.appendRuntimeEvent(REQUEST.sessionId, 'run-1', { + ...identity, + id: 'unobserved', + role: 'model', + author: 'agent', + content: { kind: 'text', text: `![image](${path})` }, + }); + await recoverChatImagePublications({ + sessionIds: [REQUEST.sessionId], + events, + artifacts: f.authority.store, + delivery: f.service, + }); + await f.service.waitForIdle(); + const saved = await events.readImmutableRuntimeEvents(REQUEST.sessionId, 'run-1'); + assert.equal(saved.filter((event) => event.id.startsWith('recovered_image_')).length, 0); + assert.equal( + ( + await f.authority.store.findImageDelivery( + REQUEST.sessionId, + REQUEST.turnId, + 'unobserved', + path, + ) + )?.status, + 'ready', + ); + } finally { + events.close(); + await f.close(); + } + }); +} + +test('queue capacity failure cannot recreate attempts after Session deletion', async () => { + const { ImageDeliveryCapacityError } = await import('@maka/storage/artifact-stores'); + const f = await fixture(); + const service = new ChatImageDeliveryService({ + ...f.ports, + artifacts: { + ...f.ports.artifacts, + putImageDeliveryJob: async () => { + f.removeSession(); + await f.authority.store.purgeSessionArtifacts(REQUEST.sessionId); + throw new ImageDeliveryCapacityError('full'); + }, + }, + }); + try { + await service.observe(REQUEST.sessionId, { + type: 'text_complete', + id: 'body', + turnId: REQUEST.turnId, + messageId: REQUEST.messageId, + ts: 1, + text: `![image](${REQUEST.source})`, + }); + assert.equal( + await f.authority.store.findImageDelivery( + REQUEST.sessionId, + REQUEST.turnId, + REQUEST.messageId, + REQUEST.source, + ), + undefined, + ); + } finally { + await service.close(); + await f.close(); + } +}); diff --git a/packages/runtime-host/src/__tests__/goal-evaluation-context.test.ts b/packages/runtime-host/src/__tests__/goal-evaluation-context.test.ts index 04574a45740..02f39218b59 100644 --- a/packages/runtime-host/src/__tests__/goal-evaluation-context.test.ts +++ b/packages/runtime-host/src/__tests__/goal-evaluation-context.test.ts @@ -730,3 +730,39 @@ test('separate form quota and massive Unicode payloads stay inside the complete 'excerpt boundaries must not split a paired surrogate', ); }); + +test('mixed tool evidence retains text and diffs while omitting images and redacting secrets', () => { + const context = buildGoalEvaluationContext([ + call('verify'), + result('verify', { + kind: 'content', + parts: [ + { kind: 'text', text: 'Verification failed: expected 5, got 3' }, + { + kind: 'image', + mimeType: 'image/png', + ref: { + kind: 'session_file', + sessionId: 'private-session', + relativePath: 'private-image', + }, + }, + { kind: 'file_diff', paths: ['answer.txt'], diff: '-expected 5\n+actual 3' }, + { kind: 'text', text: 'Authorization: Bearer secret-token-value' }, + ], + }), + ]); + assert.match(context, /Verification failed: expected 5, got 3/); + assert.match(context, /answer.txt/); + assert.match(context, /actual 3/); + assert.match(context, /\[omitted\]/); + assert.doesNotMatch(context, /private-session|private-image|secret-token-value/); + const large = buildGoalEvaluationContext([ + result('large', { + kind: 'content', + parts: [{ kind: 'text', text: '证'.repeat(8192) }], + }), + ]); + assert.ok(large.length < 1500); + assert.match(large, /truncated/); +}); diff --git a/packages/runtime-host/src/assistant/bot/avatar.ts b/packages/runtime-host/src/assistant/bot/avatar.ts index 5f3beacb00e..90a66dcbb24 100644 --- a/packages/runtime-host/src/assistant/bot/avatar.ts +++ b/packages/runtime-host/src/assistant/bot/avatar.ts @@ -90,7 +90,7 @@ export function initializeAvatar( bot.ready(); if (bot.worker() !== worker || bot.owner() !== owner) throw Error('Assistant binding changed'); - const image = validateImageBytes(result.bytes, 'chat'); + const image = validateImageBytes(result.bytes, 'avatar'); const bytes = Buffer.from(image.bytes); const imageData = bytes.toString('base64'); saved = { diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index c3caac84e94..34b67704987 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -108,7 +108,9 @@ export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. // Re-check against main before merging; sibling PRs may use the same next epoch. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 229 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 230 as const; +// 230: Tool results can contain ordered text, saved image and diff content parts. +// Older peers reject the new content variant in strict transcript decoders. // 229: Session check-in proposes cards for unfinished work across Sessions. // 228: Executor catalog queries support per-agent filtering and process-free inventory. // Older peers reject the executorId and listOnly fields in the strict query decoder. diff --git a/packages/runtime-host/src/server/chat-image-artifact.ts b/packages/runtime-host/src/server/chat-image-artifact.ts index b2ee0b7a1af..76caa66b33e 100644 --- a/packages/runtime-host/src/server/chat-image-artifact.ts +++ b/packages/runtime-host/src/server/chat-image-artifact.ts @@ -29,6 +29,7 @@ export function readyChatImageArtifact(input: { turnId: string; name: string; messageId: string; + publicationRunId?: string; source: string; image: ChatImageBytes; summary?: string; @@ -45,6 +46,7 @@ export function readyChatImageArtifact(input: { source: 'tool_result_projection', ...(input.summary !== undefined ? { summary: input.summary } : {}), imageDelivery: { + ...(input.publicationRunId ? { publicationRunId: input.publicationRunId } : {}), messageId: input.messageId, source: input.source, status: 'ready', diff --git a/packages/runtime-host/src/server/chat-image-delivery.ts b/packages/runtime-host/src/server/chat-image-delivery.ts index d8af4f4fb9f..3fda83d9102 100644 --- a/packages/runtime-host/src/server/chat-image-delivery.ts +++ b/packages/runtime-host/src/server/chat-image-delivery.ts @@ -17,7 +17,14 @@ * under the License. */ -import { createHash } from 'node:crypto'; +import { + isSessionMediaInput, + type SessionMediaInput, + type SessionContentPart, +} from '@maka/core/session-media'; +import { redactSecrets } from '@maka/core/redaction'; +import { checkedChatImage } from './chat-image-source.js'; +import { createHash, randomUUID } from 'node:crypto'; import { ImageFileReadError } from '@maka/runtime/image-file-reader'; import type { TextCompleteEvent } from '@maka/core/events'; import { @@ -28,10 +35,11 @@ import { type ImageDeliveryResult, type ImageDeliveryFailure, type ImageDeliveryIdentity, - type ImageDeliveryAttempt, } from '@maka/core/image-delivery'; import { ImageArchiveQuotaError, + ImageDeliveryCapacityError, + type ImageDeliveryJob, type InteractiveArtifactStoreWriter, } from '@maka/storage/artifact-stores'; import type { SessionAdmissionGate } from './session-admission-gate.js'; @@ -45,16 +53,28 @@ import { type ChatImageBytes, } from './chat-image-source.js'; -type DeliveryIdentity = ImageDeliveryIdentity & ImageDeliveryRequest; +type DeliveryIdentity = ImageDeliveryIdentity & + ImageDeliveryRequest & { + /** Mutable sources are captured separately for each executor publication. */ + readonly publicationId?: string; + readonly publicationRunId?: string; + }; interface DeliveryJob { - readonly done: Promise; + readonly done: Promise; run(): Promise; cancel(): void; } export interface ChatImageDeliveryPorts { readonly artifacts: Pick< InteractiveArtifactStoreWriter, - 'create' | 'findImageDelivery' | 'setImageDeliveryAttempt' + | 'create' + | 'findImageDelivery' + | 'setImageDeliveryAttempt' + | 'getInSession' + | 'putImageDeliveryJob' + | 'readImageDeliveryJob' + | 'listImageDeliveryJobs' + | 'deleteImageDeliveryJob' >; readonly admission: SessionAdmissionGate; isPresent(sessionId: string): Promise; @@ -66,6 +86,7 @@ export interface ChatImageDeliveryPorts { persistenceFailed(error: unknown): void; presentationFailed(error: unknown): void; readonly sourceReadTimeoutMs?: number; + readonly retryDelayMs?: number; readonly limits?: ImageArchiveLimits; readonly download?: typeof downloadChatImage; } @@ -75,20 +96,150 @@ export class ChatImageDeliveryService { readonly #queue: DeliveryJob[] = []; readonly #abort = new AbortController(); #running = 0; + #inlineBytes = 0; #closed = false; + #recoveryTimer?: ReturnType; + #recovering?: Promise; + readonly #submissions = new Map>(); constructor(private readonly ports: ChatImageDeliveryPorts) {} - observe(sessionId: string, event: TextCompleteEvent): void { + /** Startup barrier: recover received bytes without waiting on source I/O. */ + async recoverReceived(): Promise { + for (const job of await this.ports.artifacts.listImageDeliveryJobs(Number.MAX_SAFE_INTEGER)) { + if (this.#closed) return; + if (!(await this.ports.isPresent(job.sessionId))) { + await this.ports.artifacts.deleteImageDeliveryJob(job.id); + continue; + } + if ((await this.ports.artifacts.readImageDeliveryJob(job.id))?.bytes) + await this.#capture(job); + } + } + + /** Recovery is Host-owned and does not depend on mounting a transcript. */ + async start(): Promise { + if (this.#closed || this.#recoveryTimer) return; + await this.#recover(); + if (this.#closed) return; + this.#recoveryTimer = setInterval(() => { + void this.#recover(); + }, 1000); + this.#recoveryTimer.unref(); + } + #recover(): Promise { + if (this.#recovering) return this.#recovering; + this.#recovering = (async () => { + try { + for (const job of await this.ports.artifacts.listImageDeliveryJobs(Date.now())) { + if (this.#closed) break; + if (!(await this.ports.isPresent(job.sessionId))) { + await this.ports.artifacts.deleteImageDeliveryJob(job.id); + continue; + } + this.#schedule(job); + } + } catch (error) { + if (!this.#closed) this.ports.persistenceFailed(error); + } + })().finally(() => { + this.#recovering = undefined; + }); + return this.#recovering; + } + + /** Executor publication is Host-scoped; it never grants arbitrary client reads. */ + async publish( + owner: { sessionId: string; turnId: string; messageId: string; runId?: string }, + input: SessionMediaInput, + ): Promise { + if (!isSessionMediaInput(input)) + return { kind: 'text', text: '[Media unavailable: invalid input]' }; + const source = + input.kind === 'source' + ? input.source + : `inline:base64-sha256:${createHash('sha256').update(input.base64, 'ascii').digest('hex')}`; + const identity: DeliveryIdentity = { + ...owner, + ...(owner.runId ? { publicationRunId: owner.runId } : {}), + source, + loadRemote: true, + ...(input.kind === 'source' ? { publicationId: randomUUID() } : {}), + }; + let saved = + input.kind === 'inline' + ? await this.ports.artifacts.findImageDelivery( + owner.sessionId, + owner.turnId, + owner.messageId, + source, + ) + : undefined; + if (saved?.status !== 'ready') { + // Reserve queue capacity before decoding, including concurrent publishers waiting on metadata. + if ( + !this.#jobs.has(deliveryKey(identity)) && + input.kind === 'inline' && + this.#inlineBytes + + (input.base64.length * 3) / 4 - + (input.base64.endsWith('==') ? 2 : input.base64.endsWith('=') ? 1 : 0) > + 8 * 1024 * 1024 + ) + return { kind: 'text', text: '[Media unavailable: queue full]' }; + const bytes = + !this.#jobs.has(deliveryKey(identity)) && input.kind === 'inline' + ? Buffer.from(input.base64, 'base64') + : undefined; + if ( + bytes && + bytes.toString('base64') !== + (input as Extract).base64 + ) + return { kind: 'text', text: '[Media unavailable: invalid encoding]' }; + if (!(await this.#enqueue(identity, bytes))) + return { kind: 'text', text: '[Media unavailable: queue full]' }; + saved = await this.#waitForDelivery(identity); + } + if (saved?.status !== 'ready') + return { + kind: 'text', + text: `[Media unavailable: ${saved?.status === 'failed' ? saved.reason : 'not saved'}]`, + }; + const { record } = await this.ports.artifacts.getInSession(owner.sessionId, saved.artifactId); + if (!record?.mimeType) return { kind: 'text', text: '[Media unavailable: missing artifact]' }; + return { + kind: 'image', + mimeType: record.mimeType, + ref: { kind: 'session_file', sessionId: owner.sessionId, relativePath: saved.artifactId }, + }; + } + + async observe(sessionId: string, event: TextCompleteEvent): Promise { if (this.#closed || event.text.length > IMAGE_MARKDOWN_MAX_LENGTH) return; - // Reference destinations can grow during streaming. Only settled text - // grants local capture. Remote media loads only when a client displays it. + // Only settled canonical text grants source reads. Receiving authorized remote + // media is independent of viewport rendering, which remains lazy. + let sources: string[]; try { - for (const source of chatImageSources(event.text)) { - if (isRemoteImageSource(source)) continue; - this.#enqueue({ sessionId, turnId: event.turnId, messageId: event.messageId, source }); - } + sources = chatImageSources(event.text); } catch (error) { this.ports.presentationFailed(error); + return; + } + for (const source of sources) { + const remote = isRemoteImageSource(source); + if ( + remote && + (source.includes('') || + redactSecrets(source) !== source || + !(await this.ports.canLoadRemote(sessionId))) + ) + continue; + await this.#enqueue({ + sessionId, + turnId: event.turnId, + messageId: event.messageId, + source, + loadRemote: remote, + }); } } @@ -116,45 +267,144 @@ export class ChatImageDeliveryService { return { status: 'failed', reason: 'not_allowed' }; if (!identity.loadRemote) return { status: 'requires_confirmation' }; } - return this.#enqueue(identity) + return (await this.#enqueue(identity)) ? { status: 'pending' } : { status: 'failed', reason: 'queue_full' }; } - #enqueue(identity: DeliveryIdentity): boolean { + async #enqueue(identity: DeliveryIdentity, inline?: Uint8Array): Promise { const key = deliveryKey(identity); if (this.#jobs.has(key)) return true; - if (this.#closed || this.#jobs.size >= 128) return false; - let settle!: () => void; - const done = new Promise((resolve) => { + const pending = this.#submissions.get(key); + if (pending) return pending; + if (this.#closed || this.#inlineBytes + (inline?.byteLength ?? 0) > 8 * 1024 * 1024) + return false; + this.#inlineBytes += inline?.byteLength ?? 0; + const submission = this.ports.admission + .runOrJoin(identity.sessionId, async () => { + if (this.#closed || !(await this.ports.isPresent(identity.sessionId))) return false; + if (!identity.publicationId && !identity.retry) { + const previous = await this.ports.artifacts.findImageDelivery( + identity.sessionId, + identity.turnId, + identity.messageId, + identity.source, + ); + if (previous?.status === 'ready' || previous?.status === 'failed') return true; + } + const existing = await this.ports.artifacts.readImageDeliveryJob(key); + if (existing?.job.failure && !identity.retry) return true; + const job: ImageDeliveryJob = existing + ? { + ...existing.job, + ...(identity.retry ? { attempts: 0, nextAttemptAt: 0, failure: undefined } : {}), + } + : { + id: key, + sessionId: identity.sessionId, + turnId: identity.turnId, + messageId: identity.messageId, + source: identity.source, + ...(identity.publicationRunId ? { publicationRunId: identity.publicationRunId } : {}), + ...(identity.publicationId ? { publicationId: identity.publicationId } : {}), + loadRemote: identity.loadRemote === true, + attempts: 0, + nextAttemptAt: 0, + }; + await this.ports.artifacts.putImageDeliveryJob({ + job, + ...(inline ? { bytes: inline } : {}), + }); + this.#schedule(job); + return true; + }) + .catch(async (error) => { + if (error instanceof ImageDeliveryCapacityError) { + await this.ports.admission.runOrJoin(identity.sessionId, async () => { + if (!this.#closed && (await this.ports.isPresent(identity.sessionId))) + await this.ports.artifacts.setImageDeliveryAttempt(identity, { + status: 'failed', + reason: 'queue_full', + }); + }); + } else if (!this.#closed) this.ports.persistenceFailed(error); + return false; + }) + .finally(() => { + this.#inlineBytes -= inline?.byteLength ?? 0; + this.#submissions.delete(key); + }); + this.#submissions.set(key, submission); + return submission; + } + + #schedule(input: ImageDeliveryJob): void { + const key = input.id; + if (this.#closed || this.#jobs.has(key) || this.#jobs.size >= 128 || input.failure) return; + let settle!: (result?: ImageDeliveryResult) => void; + const done = new Promise((resolve) => { settle = resolve; }); const residency = this.ports.acquireResidency(); - const finish = () => { + const finish = (result?: ImageDeliveryResult) => { this.#jobs.delete(key); residency.release(); - settle(); + settle(result); }; const job: DeliveryJob = { done, cancel: finish, run: async () => { + let result: ImageDeliveryResult | undefined; try { - await this.#capture(identity); + result = await this.#capture(input); } catch (error) { - if (!this.#abort.signal.aborted) { - if (error instanceof ImagePersistenceError) this.ports.persistenceFailed(error.cause); - else this.ports.presentationFailed(error); - } + if (!this.#closed) this.ports.persistenceFailed(error); } finally { - finish(); + finish(result); } }, }; this.#jobs.set(key, job); this.#queue.push(job); this.#pump(); - return true; + } + + async #waitForDelivery(identity: DeliveryIdentity): Promise { + const key = deliveryKey(identity); + while (!this.#closed) { + const running = this.#jobs.get(key); + if (running) return running.done; + const staged = await this.ports.artifacts.readImageDeliveryJob(key); + if (!staged) { + const { record } = await this.ports.artifacts.getInSession(identity.sessionId, key); + return record?.imageDelivery?.status === 'ready' + ? { status: 'ready', artifactId: key } + : await this.ports.artifacts.findImageDelivery( + identity.sessionId, + identity.turnId, + identity.messageId, + identity.source, + ); + } + if (staged.job.failure) return { status: 'failed', reason: staged.job.failure }; + this.#schedule(staged.job); + if (!this.#jobs.has(key)) await this.#delay(50); + } + return undefined; + } + + async #delay(ms: number): Promise { + if (this.#closed) return; + await new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + this.#abort.signal.removeEventListener('abort', finish); + resolve(); + }; + const timer = setTimeout(finish, ms); + this.#abort.signal.addEventListener('abort', finish, { once: true }); + }); } #pump(): void { while (!this.#closed && this.#running < 2 && this.#queue.length) { @@ -166,80 +416,117 @@ export class ChatImageDeliveryService { }); } } - async #capture(identity: DeliveryIdentity): Promise { - const { sessionId, turnId, messageId, source } = identity; - const existing = await this.ports.artifacts.findImageDelivery( - sessionId, - turnId, - messageId, - source, - ); - if (existing?.status === 'ready' || (existing?.status === 'failed' && !identity.retry)) return; - const metadata = { messageId, source }; - const resultId = `chat_image_result_${deliveryKey(identity)}`; - const publish = async (write: () => Promise) => - this.ports.admission.runOrJoin(sessionId, async () => { - if (this.#abort.signal.aborted || !(await this.ports.isPresent(sessionId))) return; - try { - await write(); - } catch (error) { - if (error instanceof ImageArchiveQuotaError) throw error; - throw new ImagePersistenceError('Image persistence failed', { cause: error }); - } - }); - const publishAttempt = (attempt: ImageDeliveryAttempt) => - publish(() => this.ports.artifacts.setImageDeliveryAttempt(identity, attempt)); - const publishFailure = (reason: ImageDeliveryFailure) => - publishAttempt({ status: 'failed', reason }); - await publishAttempt({ status: 'pending' }); - if (this.#abort.signal.aborted || !(await this.ports.isPresent(sessionId))) return; - const readAbort = new AbortController(); - // Keep the read deadline alive even when a source has no active handles. - const deadline = setTimeout(() => readAbort.abort(), this.ports.sourceReadTimeoutMs ?? 10_000); - const signal = AbortSignal.any([this.#abort.signal, readAbort.signal]); - let image: ChatImageBytes; - try { - const path = localImagePath(source); - if (path !== undefined) { - image = await abortable(() => this.ports.readLocalImage(sessionId, path, signal), signal); - } else { - if (!identity.loadRemote || !(await this.ports.canLoadRemote(sessionId))) - throw new ImageSourceError('not_allowed'); - image = await (this.ports.download ?? downloadChatImage)(source, signal); - } - } catch (error) { - if (this.#abort.signal.aborted) return; - await publishFailure(failureReason(error, source)); - return; - } finally { - clearTimeout(deadline); + async #capture(initial: ImageDeliveryJob): Promise { + let job = initial; + const { sessionId, turnId, messageId, source } = job; + const saved = await this.ports.artifacts.getInSession(sessionId, job.id); + if (saved.record?.imageDelivery?.status === 'ready') { + await this.ports.artifacts.deleteImageDeliveryJob(job.id); + return { status: 'ready', artifactId: job.id }; } - if (this.#abort.signal.aborted) return; - if (signal.aborted) { - await publishFailure(failureReason(signal.reason, source)); - return; - } - try { - await publish(() => - this.ports.artifacts.create( - readyChatImageArtifact({ - id: resultId, - sessionId, - turnId, - name: 'chat-image', - ...metadata, - image, - limits: this.ports.limits, - }), - ), + const failed = async (reason: ImageDeliveryFailure): Promise => { + await this.ports.admission.runOrJoin(sessionId, async () => { + if (!this.#closed && (await this.ports.isPresent(sessionId))) + await this.ports.artifacts.putImageDeliveryJob({ job: { ...job, failure: reason } }); + }); + return { status: 'failed', reason }; + }; + while (!this.#closed && (await this.ports.isPresent(sessionId))) { + if (job.nextAttemptAt > Date.now()) await this.#delay(job.nextAttemptAt - Date.now()); + if (this.#closed) return; + const staged = await this.ports.artifacts.readImageDeliveryJob(job.id); + if (!staged) return; + let image: ChatImageBytes; + const readAbort = new AbortController(); + const deadline = setTimeout( + () => readAbort.abort(), + this.ports.sourceReadTimeoutMs ?? 10_000, ); - } catch (error) { - if (!(error instanceof ImageArchiveQuotaError)) throw error; - await publishFailure('quota_exceeded'); + const signal = AbortSignal.any([this.#abort.signal, readAbort.signal]); + try { + if (staged.bytes) image = checkedChatImage(staged.bytes); + else { + if (job.attempts >= 3) + return await failed(isRemoteImageSource(source) ? 'download_failed' : 'read_failed'); + job = { ...job, attempts: job.attempts + 1 }; + let accepted = false; + await this.ports.admission.runOrJoin(sessionId, async () => { + if (this.#closed || !(await this.ports.isPresent(sessionId))) return; + await this.ports.artifacts.putImageDeliveryJob({ job }); + accepted = true; + }); + if (!accepted) return; + const path = localImagePath(source); + if (path !== undefined) + image = await abortable( + () => this.ports.readLocalImage(sessionId, path, signal), + signal, + ); + else { + if (!job.loadRemote || !(await this.ports.canLoadRemote(sessionId))) + throw new ImageSourceError('not_allowed'); + image = await (this.ports.download ?? downloadChatImage)(source, signal); + } + } + } catch (error) { + if (this.#closed) return; + const reason = failureReason(error, source); + if ( + ['download_failed', 'read_failed'].includes(reason) && + job.attempts < 3 && + !staged.bytes + ) { + job = { + ...job, + nextAttemptAt: + Date.now() + (this.ports.retryDelayMs ?? 500) * 2 ** Math.max(0, job.attempts - 1), + }; + await this.ports.admission.runOrJoin(sessionId, async () => { + if (!this.#closed && (await this.ports.isPresent(sessionId))) + await this.ports.artifacts.putImageDeliveryJob({ job }); + }); + continue; + } + return await failed(reason); + } finally { + clearTimeout(deadline); + } + if (this.#closed) return; + try { + let committed = false; + await this.ports.admission.runOrJoin(sessionId, async () => { + if (this.#closed || !(await this.ports.isPresent(sessionId))) return; + // Keep received bytes until publication commits. Restart never needs the + // original temporary path or URL once this durable spool is written. + await this.ports.artifacts.putImageDeliveryJob({ job, bytes: image.bytes }); + await this.ports.artifacts.create( + readyChatImageArtifact({ + id: job.id, + ...(job.publicationRunId ? { publicationRunId: job.publicationRunId } : {}), + sessionId, + turnId, + messageId, + source, + name: 'chat-image', + image, + limits: this.ports.limits, + }), + ); + committed = true; + }); + return committed ? { status: 'ready', artifactId: job.id } : undefined; + } catch (error) { + if (error instanceof ImageArchiveQuotaError) return failed('quota_exceeded'); + if (error instanceof ImageDeliveryCapacityError) return failed('queue_full'); + throw error; + } } + if (!this.#closed) await this.ports.artifacts.deleteImageDeliveryJob(job.id); + return undefined; } beginDrain(): void { this.#closed = true; + clearInterval(this.#recoveryTimer); this.#abort.abort(); for (const job of this.#queue.splice(0)) job.cancel(); } @@ -248,14 +535,26 @@ export class ChatImageDeliveryService { await this.waitForIdle(); } async waitForIdle(): Promise { + await Promise.all([...this.#submissions.values()]); + await this.#recovering; await Promise.all([...this.#jobs.values()].map((job) => job.done)); } } -class ImagePersistenceError extends Error {} function deliveryKey(i: DeliveryIdentity): string { - return createHash('sha256') - .update(JSON.stringify([i.sessionId, i.turnId, i.messageId, i.source])) - .digest('hex'); + return ( + 'chat_image_result_' + + createHash('sha256') + .update( + JSON.stringify([ + i.sessionId, + i.turnId, + i.messageId, + i.source, + ...(i.publicationId ? [i.publicationId] : []), + ]), + ) + .digest('hex') + ); } function failureReason(error: unknown, source: string): ImageDeliveryFailure { if (error instanceof ImageSourceError) return error.reason; diff --git a/packages/runtime-host/src/server/chat-image-recovery.ts b/packages/runtime-host/src/server/chat-image-recovery.ts new file mode 100644 index 00000000000..9152439147c --- /dev/null +++ b/packages/runtime-host/src/server/chat-image-recovery.ts @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import { formatAttachmentResourceRef } from '@maka/core/attachments'; +import type { RuntimeEvent } from '@maka/core/runtime-event'; +import type { ToolResultContent } from '@maka/core/events'; +import type { InteractiveArtifactStoreWriter } from '@maka/storage/artifact-stores'; +import type { ExecutionStoresWriter } from '@maka/storage/execution-stores'; +import { markPersisted } from '@maka/core/persisted-value'; +import { decodePersistedToolResultContent } from '@maka/core/tool-result-record-schema'; +import { IMAGE_MARKDOWN_MAX_LENGTH } from '@maka/core/image-delivery'; +import { chatImageSources } from './chat-image-markdown.js'; +import type { ChatImageDeliveryService } from './chat-image-delivery.js'; + +/** Runs before interrupted invocations are sealed. Never reopens a terminal run. */ +export async function recoverChatImagePublications(input: { + sessionIds: readonly string[]; + events: Pick< + ExecutionStoresWriter<'interactive'>['runtimeEventStore'], + 'listInvocationRecoveryInventory' | 'scanRuntimeEvents' | 'appendRuntimeEvent' + >; + artifacts: Pick; + delivery: Pick; + onLimited?(sessionId: string, runId: string): void; +}): Promise { + // Only already-received bytes cross this barrier. Remote/local source reads + // remain background work and cannot hold up Host startup. + await input.delivery.recoverReceived(); + const inventory = await input.events.listInvocationRecoveryInventory(input.sessionIds); + for (const { candidate: run } of inventory) { + if (!run || run.terminalEvent) continue; + const artifacts = (await input.artifacts.listTurnArtifacts(run.sessionId, run.turnId)).filter( + (artifact) => artifact.imageDelivery?.publicationRunId === run.runId, + ); + const targets = artifacts.map((artifact) => ({ + artifact, + source: formatAttachmentResourceRef({ + kind: 'session_file', + sessionId: run.sessionId, + relativePath: artifact.id, + })!, + id: + 'recovered_image_' + + createHash('sha256') + .update(JSON.stringify([run.runId, artifact.id])) + .digest('hex'), + })); + const texts: RuntimeEvent[] = []; + const referenced = new Set(); + const collect = (content: ToolResultContent): void => { + if ( + content.kind === 'image' && + content.ref.kind === 'session_file' && + content.ref.sessionId === run.sessionId + ) + referenced.add(content.ref.relativePath); + if (content.kind === 'content') for (const part of content.parts) collect(part); + }; + const scanned = await input.events.scanRuntimeEvents( + run.sessionId, + run.runId, + { + maxBatchBytes: 8 * 1024 * 1024, + maxRecordBytes: 64 * 1024 * 1024, + maxImmutableRecords: 1_000_000, + maxImmutableBytes: Number.MAX_SAFE_INTEGER, + maxPartialRecords: 65536, + maxPartialBytes: 256 * 1024 * 1024, + }, + (batch) => { + for (const event of batch) { + if (event.partial) continue; + for (const target of targets) { + if ( + event.id === target.id || + (event.role === 'model' && + event.content?.kind === 'text' && + event.content.text.includes(target.source)) + ) + referenced.add(target.artifact.id); + } + if (event.content?.kind === 'function_response') { + try { + collect( + decodePersistedToolResultContent( + markPersisted(event.content.result), + ), + ); + } catch { + /* An opaque legacy result grants no image reference. */ + } + } + if ( + event.role === 'model' && + event.content?.kind === 'text' && + event.content.text.length <= IMAGE_MARKDOWN_MAX_LENGTH && + chatImageSources(event.content.text).length + ) { + texts.push(event); + if (texts.length > 512) texts.shift(); + } + } + }, + ); + // An oversized individual record must not block unrelated Host recovery. + // Do not infer a missing publication from incomplete evidence. + if (scanned.status !== 'complete') { + input.onLimited?.(run.sessionId, run.runId); + continue; + } + for (const event of texts) { + if (event.content?.kind !== 'text') continue; + // Covers the crash window between canonical text commit and observation. + await input.delivery.observe(run.sessionId, { + type: 'text_complete', + id: event.id, + turnId: event.turnId, + ts: event.ts, + messageId: event.refs?.storedMessageId ?? event.refs?.providerEventId ?? event.id, + text: event.content.text, + }); + } + for (const { artifact, source, id } of targets) { + if (referenced.has(artifact.id)) continue; + // A durable Artifact is also the publication receipt: it retains exact run + // ownership after the spool is retired. Stable identity makes replay safe + // if the process dies again after this event commits. + const recovered: RuntimeEvent = { + id, + sessionId: run.sessionId, + runId: run.runId, + invocationId: run.invocationId, + turnId: run.turnId, + ts: artifact.createdAt, + partial: false, + role: 'model', + author: 'host', + modelVisibility: 'hidden', + content: { + kind: 'text', + text: `Recovered image received before interruption.\n\n![Recovered image](${source})`, + }, + refs: { artifactId: artifact.id, storedMessageId: id }, + }; + await input.events.appendRuntimeEvent(run.sessionId, run.runId, recovered, { durable: true }); + } + } +} diff --git a/packages/runtime-host/src/server/chat-image-source.ts b/packages/runtime-host/src/server/chat-image-source.ts index 34596f90732..a45e53e71aa 100644 --- a/packages/runtime-host/src/server/chat-image-source.ts +++ b/packages/runtime-host/src/server/chat-image-source.ts @@ -18,7 +18,7 @@ */ import { fileURLToPath } from 'node:url'; -import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; +import { IMAGE_ARCHIVE_MAX_BYTES } from '@maka/core/image-delivery'; import { ImageFileError, imageFileFailureReason, @@ -94,7 +94,7 @@ export async function downloadChatImage( await response.body?.cancel(); throw new ImageSourceError('download_failed'); } - if (Number(response.headers.get('content-length')) > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) { + if (Number(response.headers.get('content-length')) > IMAGE_ARCHIVE_MAX_BYTES) { await response.body?.cancel(); throw new ImageSourceError('too_large'); } @@ -107,7 +107,7 @@ export async function downloadChatImage( const chunk = await reader.read(); if (chunk.done) break; size += chunk.value.length; - if (size > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) throw new ImageSourceError('too_large'); + if (size > IMAGE_ARCHIVE_MAX_BYTES) throw new ImageSourceError('too_large'); chunks.push(Buffer.from(chunk.value)); } } finally { diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index 83d743f6347..b3b238c6ddb 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -22,6 +22,7 @@ import { readMatterRuntimeEvents } from '../assistant/proactive-matters/settleme import { markAssistantBackgroundSessions } from '../assistant/background-sessions.js'; import { PersonalAssistantService, ASSISTANT_LEGACY_PACKAGES } from '../assistant/service.js'; import { assistantInputTurnId, legacyAssistantInputTurnId } from '../assistant/bot/turn-ids.js'; +import { recoverChatImagePublications } from './chat-image-recovery.js'; import { createAssistantMessageReader } from './session-transcript-reader.js'; import { createImageFileReader } from '@maka/runtime/image-file-reader'; import { ChatImageDeliveryService } from './chat-image-delivery.js'; @@ -1494,6 +1495,7 @@ export async function createExecutionRuntimeHostComposition( : {}), ...(factoryContext.systemPrompt ? { instructions: factoryContext.systemPrompt } : {}), binding, + publishMedia: (owner, media) => imageDelivery!.publish(owner, media), }), }; }, @@ -3691,6 +3693,21 @@ export async function createExecutionRuntimeHostComposition( recovery: { executions: async () => { await coordinator.prepareRecovery(); + if (imageDelivery) { + await recoverChatImagePublications({ + sessionIds: recoverySessions + .filter((session) => !session.isArchived) + .map((session) => session.id), + events: stores.runtimeEventStore, + artifacts: openedArtifactStore!, + delivery: imageDelivery, + onLimited: () => + console.warn( + '[runtime-host] Image publication recovery skipped an oversized event; archived bytes are retained', + ), + }); + await imageDelivery.start(); + } await interactions.recoverPendingAfterHostRestart(); await requireSessionManager(manager).recoverInterruptedSessionsStrict(stores); await requireSessionManager(manager).recoverChildWorkspacePatches( diff --git a/packages/runtime-host/src/server/goal-evaluation-context.ts b/packages/runtime-host/src/server/goal-evaluation-context.ts index 0b5b99c6134..64c112b06f5 100644 --- a/packages/runtime-host/src/server/goal-evaluation-context.ts +++ b/packages/runtime-host/src/server/goal-evaluation-context.ts @@ -203,6 +203,11 @@ function formatToolEvidence({ call, result }: ToolEvidence, withholdResult: bool /** Only human-readable result fields; never raw provider replay or image data. */ function projectResult(content: ToolResultContent, toolName: string | undefined): unknown { switch (content.kind) { + case 'content': + return { + kind: content.kind, + parts: content.parts.map((part) => projectResult(part, toolName)), + }; case 'text': return { kind: content.kind, text: content.text, uncertainOutcome: content.uncertainOutcome }; case 'json': { diff --git a/packages/runtime-host/src/server/root-turn-coordinator.ts b/packages/runtime-host/src/server/root-turn-coordinator.ts index 1f8bd113580..c1ee2b1a87f 100644 --- a/packages/runtime-host/src/server/root-turn-coordinator.ts +++ b/packages/runtime-host/src/server/root-turn-coordinator.ts @@ -391,7 +391,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { private readonly observeAssistantText?: ( sessionId: string, event: import('@maka/core/events').TextCompleteEvent, - ) => void, + ) => void | Promise, ) { this.stores = authenticateExecutionStoresWriter(stores, 'interactive'); this.executionProjection = new HostedExecutionProjectionReader(this.stores); @@ -3274,7 +3274,7 @@ export class RootTurnCoordinator implements HostedExecutionAuthority { } if (event.type === 'text_complete') { try { - this.observeAssistantText?.(input.sessionId, event); + await this.observeAssistantText?.(input.sessionId, event); } catch { // Image presentation must never interrupt canonical execution. } diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index a365090b7df..c5c8dd29e84 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -4210,3 +4210,57 @@ function invocationOpenedEvent(run: SeededRun = {}): RuntimeEvent { opening: invocationOpening(run), }); } + +test('conversation copies collect and rewrite every image in ordered media content', () => { + const message: StoredMessage = { + type: 'tool_result', + id: 'result-media', + turnId: 'turn-1', + ts: 1, + toolUseId: 'tool-media', + isError: false, + content: { + kind: 'content', + parts: [ + { kind: 'text', text: 'before ![](maka://runtime/attachments/svg-original)' }, + { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session-source', relativePath: 'svg-original' }, + }, + { kind: 'text', text: 'after' }, + ], + }, + }; + assert.deepEqual( + [ + ...collectConversationCopySessionFileRefs({ + sourceSessionId: 'session-source', + messages: [message], + runtimeEvents: [], + }), + ], + ['svg-original'], + ); + const copied = rewriteConversationCopyMessage(message, { + sourceSessionId: 'session-source', + targetSessionId: 'session-target', + artifactIds: new Map([['svg-original', 'svg-copy']]), + relativePaths: new Map(), + runtimeEventIds: new Map(), + runIds: new Map(), + providerTraceIds: new Map(), + }); + assert.equal(copied.type, 'tool_result'); + if (copied.type !== 'tool_result' || copied.content.kind !== 'content') + throw new Error('Expected media'); + assert.deepEqual(copied.content.parts[1], { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: 'session-target', relativePath: 'svg-copy' }, + }); + assert.deepEqual(copied.content.parts[0], { + kind: 'text', + text: 'before ![](maka://runtime/attachments/svg-copy)', + }); +}); diff --git a/packages/runtime/src/__tests__/image-file.test.ts b/packages/runtime/src/__tests__/image-file.test.ts index 77010595bbc..5a5ddb5b7be 100644 --- a/packages/runtime/src/__tests__/image-file.test.ts +++ b/packages/runtime/src/__tests__/image-file.test.ts @@ -27,7 +27,7 @@ import { readWorkspaceFile, validateImageBytes } from '../image-file.js'; import { LocalWorkspaceExecutor } from '../workspace-executor.js'; import { executeFilesystemOperation } from '../filesystem-worker/operations.js'; import { MAX_READ_IMAGE_BYTES } from '@maka/core/attachments'; -import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; +import { IMAGE_ARCHIVE_MAX_BYTES } from '@maka/core/image-delivery'; import { createImageFileReader, ImageFileReadError } from '../image-file-reader.js'; import { FilesystemWorkerClientError } from '../filesystem-worker/client.js'; import { executeFilesystemWorkerRequest } from '../filesystem-worker/operations.js'; @@ -103,7 +103,7 @@ test('Worker transports typed image validation failures instead of generic files const path = join(cwd, 'image.png'); for (const [content, expected] of [ [ONE_PIXEL_PNG.subarray(0, 8), 'invalid_image'], - [Buffer.alloc(ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + 1), 'image_too_large'], + [Buffer.alloc(IMAGE_ARCHIVE_MAX_BYTES + 1), 'image_too_large'], ] as const) { await writeFile(path, content); const response = await executeFilesystemWorkerRequest({ @@ -152,8 +152,8 @@ for (const route of ['workspace', 'worker'] as const) { assert.ok('bytes' in result && result.bytes instanceof Uint8Array); assert.deepEqual(Buffer.from(result.bytes), WIDE_PNG); } - await truncate(path, ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + 1); - await assert.rejects(read('chat'), /2 MiB/); + await truncate(path, IMAGE_ARCHIVE_MAX_BYTES + 1); + await assert.rejects(read('chat'), /8 MiB/); await writeFile(path, 'not an image'); await assert.rejects(read('chat'), /not a supported/i); }); diff --git a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts index 822b4b6c96b..64836f84a23 100644 --- a/packages/runtime/src/__tests__/plugin-executor-backend.test.ts +++ b/packages/runtime/src/__tests__/plugin-executor-backend.test.ts @@ -649,3 +649,55 @@ async function collect(events: AsyncIterable): Promise { + const root = new Context(); + const service = new PluginExecutorService(root); + const publications: unknown[] = []; + root + .extend({ + maka: { rootId: 'profile', packageId: 'fixture', entryId: 'provider', generation: 1 }, + }) + .executors.register({ + id: 'media', + capabilities: { toolActivity: true }, + execute: async (_request, context) => { + context.emit({ type: 'tool_start', toolCallId: 'draw', name: 'draw' }); + const image = await context.publishMedia!({ + kind: 'inline', + mimeType: 'image/svg+xml', + base64: 'PHN2Zy8+', + }); + context.emit({ + type: 'tool_result', + toolCallId: 'draw', + content: { kind: 'content', parts: [{ kind: 'text', text: 'drawing' }, image] }, + }); + return { status: 'completed', text: 'done' }; + }, + }); + const backend = new PluginExecutorBackend({ + sessionId: 'session-a', + cwd: '/workspace', + binding: service.bind('session-a', 'media'), + publishMedia: async (owner, input) => { + publications.push({ owner, input }); + return { + kind: 'image', + mimeType: 'image/svg+xml', + ref: { kind: 'session_file', sessionId: owner.sessionId, relativePath: 'saved-svg' }, + }; + }, + }); + try { + const events = await collect(backend.send({ turnId: 'turn-a', text: 'draw' })); + assert.equal(publications.length, 1); + const result = events.find((event) => event.type === 'tool_result'); + assert.equal(result?.content.kind, 'content'); + assert.doesNotMatch(JSON.stringify(events), /PHN2Zy8\+/); + assert.match(JSON.stringify(events), /saved-svg/); + } finally { + await backend.dispose(); + await root.fiber.dispose(); + } +}); diff --git a/packages/runtime/src/conversation-copy.ts b/packages/runtime/src/conversation-copy.ts index ce29a964e1c..2c8a23f60b1 100644 --- a/packages/runtime/src/conversation-copy.ts +++ b/packages/runtime/src/conversation-copy.ts @@ -191,6 +191,7 @@ function collectConversationCopyStorageRefs( const refs: StorageRef[] = []; const addContent = (content: ToolResultContent): void => { if (content.kind === 'image') refs.push(content.ref); + if (content.kind === 'content') for (const part of content.parts) addContent(part); }; const addSerialized = (value: unknown): void => { if (isArchivedToolResultPlaceholder(value)) return; @@ -1892,6 +1893,23 @@ function rewriteToolResultContent( return content; } } + if (content.kind === 'content') { + return { + ...content, + parts: content.parts.map((part) => { + if (part.kind === 'text') + return { + ...part, + text: rewriteAttachmentResourceRefs(part.text, references.artifactIds), + }; + if (part.kind !== 'image') return part; + const ref = rewriteStorageRef(part.ref, references); + if (ref.kind !== 'session_file') + throw new Error('Copied media must retain a Session artifact reference'); + return { ...part, ref }; + }), + }; + } if (content.kind === 'image') { return { ...content, ref: rewriteStorageRef(content.ref, references) }; } diff --git a/packages/runtime/src/image-file.ts b/packages/runtime/src/image-file.ts index c1f0ec5a71f..a4a0c4f2483 100644 --- a/packages/runtime/src/image-file.ts +++ b/packages/runtime/src/image-file.ts @@ -29,6 +29,7 @@ import { sniffAttachmentMimeType, } from '@maka/core/attachments'; import { ARTIFACT_IMAGE_PREVIEW_MAX_BYTES } from '@maka/core/artifacts'; +import { IMAGE_ARCHIVE_MAX_BYTES } from '@maka/core/image-delivery'; export interface WorkspaceFileReadOptions { imagePurpose?: 'chat'; @@ -72,7 +73,7 @@ export async function readWorkspaceFile( if (options.imagePurpose === 'chat') { const size = await file.stat(); if (!size.isFile()) throw new Error('Image path is not a file.'); - if (size.size > ARTIFACT_IMAGE_PREVIEW_MAX_BYTES) throw imageTooLargeError('chat'); + if (size.size > IMAGE_ARCHIVE_MAX_BYTES) throw imageTooLargeError('chat'); return validateImageBytes(await file.readFile({ signal: options.abortSignal }), 'chat'); } const prefix = Buffer.alloc(ATTACHMENT_MIME_SNIFF_BYTES); @@ -92,12 +93,19 @@ export async function readWorkspaceFile( export function validateImageBytes( bytes: Uint8Array, - purpose: 'model' | 'chat' = 'model', + purpose: 'model' | 'chat' | 'avatar' = 'model', ): { bytes: Uint8Array; mimeType: ImageMimeType; } { - if (bytes.length > (purpose === 'chat' ? ARTIFACT_IMAGE_PREVIEW_MAX_BYTES : MAX_READ_IMAGE_BYTES)) + if ( + bytes.length > + (purpose === 'chat' + ? IMAGE_ARCHIVE_MAX_BYTES + : purpose === 'avatar' + ? ARTIFACT_IMAGE_PREVIEW_MAX_BYTES + : MAX_READ_IMAGE_BYTES) + ) throw imageTooLargeError(purpose); const mimeType = sniffImageMime(bytes); if (!mimeType) @@ -128,7 +136,7 @@ export function validateImageBytes( // Compressed byte size does not bound renderer decoding memory. Permit large // screenshots while rejecting pathological headers before archival/preview. if ( - purpose === 'chat' && + purpose !== 'model' && (Math.max(dimensions.width, dimensions.height) > 16_384 || dimensions.width * dimensions.height > 32 * 1024 * 1024) ) { @@ -137,12 +145,14 @@ export function validateImageBytes( return { bytes, mimeType }; } -function imageTooLargeError(purpose: 'model' | 'chat' = 'model'): Error { +function imageTooLargeError(purpose: 'model' | 'chat' | 'avatar' = 'model'): Error { return new ImageFileError( 'ERR_IMAGE_TOO_LARGE', - purpose === 'chat' - ? 'Image exceeds a chat preview limit (2 MiB, 16384px per edge, or 32 megapixels); resize it before publishing.' - : READ_IMAGE_TOO_LARGE_MESSAGE, + purpose === 'avatar' + ? 'Avatar exceeds a preview limit (2 MiB, 16384px per edge, or 32 megapixels).' + : purpose === 'chat' + ? `Image exceeds a chat archive limit (${IMAGE_ARCHIVE_MAX_BYTES / (1024 * 1024)} MiB, 16384px per edge, or 32 megapixels); resize it before publishing.` + : READ_IMAGE_TOO_LARGE_MESSAGE, ); } diff --git a/packages/runtime/src/plugin-executor-backend.ts b/packages/runtime/src/plugin-executor-backend.ts index 16083e7080b..bcaa93c08cb 100644 --- a/packages/runtime/src/plugin-executor-backend.ts +++ b/packages/runtime/src/plugin-executor-backend.ts @@ -61,6 +61,10 @@ export interface PluginExecutorBackendInput { readonly model?: string; readonly thinkingLevel?: ThinkingLevel; readonly binding: PluginExecutorBinding; + readonly publishMedia?: ( + owner: { sessionId: string; turnId: string; messageId: string; runId?: string }, + input: import('@maka/core/session-media').SessionMediaInput, + ) => Promise; readonly newId?: () => string; readonly now?: () => number; } @@ -75,6 +79,7 @@ export class PluginExecutorBackend implements AgentBackend { readonly #model?: string; readonly #thinkingLevel?: ThinkingLevel; readonly #binding: PluginExecutorBinding; + readonly #publishMedia: PluginExecutorBackendInput['publishMedia']; readonly #newId: () => string; readonly #now: () => number; readonly #active = new Set(); @@ -88,6 +93,7 @@ export class PluginExecutorBackend implements AgentBackend { this.#model = input.model; this.#thinkingLevel = input.thinkingLevel; this.#binding = input.binding; + this.#publishMedia = input.publishMedia; this.#newId = input.newId ?? randomUUID; this.#now = input.now ?? Date.now; } @@ -209,6 +215,20 @@ export class PluginExecutorBackend implements AgentBackend { }, { signal, + ...(this.#publishMedia + ? { + onPublishMedia: (media: import('@maka/core/session-media').SessionMediaInput) => + this.#publishMedia!( + { + sessionId: this.sessionId, + turnId, + messageId, + ...(input.runId ? { runId: input.runId } : {}), + }, + media, + ), + } + : {}), onSettled: () => { settled = true; }, @@ -571,9 +591,11 @@ export class PluginExecutorBackend implements AgentBackend { providerExecuted: true, isError: event.isError ?? false, content: - event.content.kind === 'text' - ? event.content - : { kind: 'file_diff', paths: [...event.content.paths], diff: event.content.diff }, + event.content.kind === 'content' + ? { kind: 'content', parts: [...event.content.parts] } + : event.content.kind === 'text' + ? event.content + : { kind: 'file_diff', paths: [...event.content.paths], diff: event.content.diff }, }); } diff --git a/packages/runtime/src/plugin-executor-service.ts b/packages/runtime/src/plugin-executor-service.ts index 5de32b14242..af77ad24327 100644 --- a/packages/runtime/src/plugin-executor-service.ts +++ b/packages/runtime/src/plugin-executor-service.ts @@ -17,6 +17,13 @@ * under the License. */ +import { + isSessionMediaInput, + isSessionContentPart, + SESSION_CONTENT_MAX_PARTS, + type SessionMediaInput, + type SessionContentPart, +} from '@maka/core/session-media'; import { isExecutorConfiguration, normalizeCatalogEntry, @@ -105,6 +112,7 @@ export type PluginExecutorOutputEvent = /** Durable tool-result shapes that an external executor may publish directly. */ export type PluginExecutorToolResultContent = + | { readonly kind: 'content'; readonly parts: readonly SessionContentPart[] } | { readonly kind: 'text'; readonly text: string } | { readonly kind: 'file_diff'; readonly paths: readonly string[]; readonly diff: string }; @@ -147,6 +155,7 @@ export type PluginExecutorResult = export interface PluginExecutorContext { readonly signal: AbortSignal; emit(event: PluginExecutorOutputEvent): void; + publishMedia?(input: SessionMediaInput): Promise; /** Request one provider-defined permission choice through Maka's hosted form authority. */ requestPermission( request: PluginExecutorPermissionRequest, @@ -190,6 +199,7 @@ export interface PluginExecutorProvider { } export interface PluginExecutorExecutionOptions { + readonly onPublishMedia?: (input: SessionMediaInput) => Promise; readonly signal?: AbortSignal; /** Called when the provider returns a terminal result, before the Runtime validates it. */ readonly onSettled?: () => void; @@ -549,12 +559,33 @@ export class PluginExecutorService extends Service { emit: (event) => { if (entry.retired) return; const normalized = normalizeOutputEvent(event, entry.provider.capabilities); + if ( + normalized.type === 'tool_result' && + normalized.content.kind === 'content' && + normalized.content.parts.some( + (part) => + part.kind === 'image' && part.ref.sessionId !== normalizedRequest.sessionId, + ) + ) + throw new TypeError('Executor media is outside the current Session'); try { options.onEvent?.(normalized); } catch { // A presentation observer must not change external execution. } }, + publishMedia: async (input) => { + if (signal.aborted || entry.retired) + return { kind: 'text', text: '[Media unavailable: execution ended]' }; + if (!isSessionMediaInput(input)) + return { kind: 'text', text: '[Media unavailable: invalid input]' }; + const part = (await options.onPublishMedia?.(Object.freeze({ ...input }))) ?? { + kind: 'text' as const, + text: '[Media preview unavailable]', + }; + if (!isSessionContentPart(part)) throw new TypeError('Invalid published media'); + return part; + }, requestPermission: async (request) => { if (signal.aborted || entry.retired) return Object.freeze({ outcome: 'cancelled' }); const normalized = normalizePermissionRequest(request); @@ -778,13 +809,20 @@ function normalizeOutputEvent( type: event.type, toolCallId: event.toolCallId, content: - event.content.kind === 'text' - ? Object.freeze({ kind: 'text' as const, text: event.content.text }) - : Object.freeze({ - kind: 'file_diff' as const, - paths: Object.freeze([...event.content.paths]), - diff: event.content.diff, - }), + event.content.kind === 'content' + ? Object.freeze({ + kind: 'content' as const, + parts: Object.freeze( + event.content.parts.map((part) => Object.freeze(structuredClone(part))), + ), + }) + : event.content.kind === 'text' + ? Object.freeze({ kind: 'text' as const, text: event.content.text }) + : Object.freeze({ + kind: 'file_diff' as const, + paths: Object.freeze([...event.content.paths]), + diff: event.content.diff, + }), ...(event.isError === undefined ? {} : { isError: event.isError }), }); } @@ -793,6 +831,14 @@ function normalizeOutputEvent( function isPluginToolResultContent(value: PluginExecutorToolResultContent): boolean { if (!value || typeof value !== 'object') return false; + if (value.kind === 'content') + return ( + Array.isArray(value.parts) && + value.parts.length > 0 && + value.parts.length <= SESSION_CONTENT_MAX_PARTS && + value.parts.every(isSessionContentPart) && + Buffer.byteLength(JSON.stringify(value.parts), 'utf8') <= 1024 * 1024 + ); if (value.kind === 'text') return isSafeEventText(value.text); return ( value.kind === 'file_diff' && diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index 0ab11a1756c..d31244abf32 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -3469,6 +3469,7 @@ export function deriveToolResultStatus( case 'file_diff': case 'file_write': case 'image': + case 'content': case 'summary': case 'archived_tool_result': case 'web_search': diff --git a/packages/storage/src/__tests__/image-delivery-storage.test.ts b/packages/storage/src/__tests__/image-delivery-storage.test.ts index f3a10a734a9..860f25a4876 100644 --- a/packages/storage/src/__tests__/image-delivery-storage.test.ts +++ b/packages/storage/src/__tests__/image-delivery-storage.test.ts @@ -40,7 +40,7 @@ const lookup = ( i = identity, ) => store.findImageDelivery(i.sessionId, i.turnId, i.messageId, i.source); -test('schema 3 upgrades directly to 4 without rewriting existing artifacts', async () => { +test('schema 3 upgrades directly to 5 without rewriting existing artifacts', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-image-schema-')); let authority = createSqliteArtifactStoreWriteAuthority(root); try { @@ -56,7 +56,7 @@ test('schema 3 upgrades directly to 4 without rewriting existing artifacts', asy authority.close(); const db = new DatabaseSync(join(root, 'runtime.sqlite')); try { - db.exec(`DROP TABLE image_delivery_attempts; DROP INDEX artifact_image_delivery_lookup; + db.exec(`DROP TABLE image_delivery_jobs; DROP TABLE image_delivery_attempts; DROP INDEX artifact_image_delivery_lookup; UPDATE operational_schema_migrations SET version = 3 WHERE scope = 'artifact';`); } finally { db.close(); @@ -74,7 +74,7 @@ test('schema 3 upgrades directly to 4 without rewriting existing artifacts', asy reopened .prepare("SELECT version FROM operational_schema_migrations WHERE scope = 'artifact'") .get()?.version, - 4, + 5, ); assert.equal( reopened.prepare('SELECT COUNT(*) AS n FROM image_delivery_attempts').get()?.n, @@ -212,3 +212,170 @@ test('archive policy enforces defaults even when a caller omits its limits', () }), ); }); + +test('received bytes survive reopening, stay immutable, and retire atomically on publication', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-spool-')); + let authority = createSqliteArtifactStoreWriteAuthority(root); + const bytes = new Uint8Array([1, 2, 3]); + const job = { + ...identity, + id: 'durable-reception', + loadRemote: false, + attempts: 1, + nextAttemptAt: 0, + }; + try { + await authority.store.putImageDeliveryJob({ job, bytes }); + authority.close(); + authority = createSqliteArtifactStoreWriteAuthority(root); + const staged = await authority.store.readImageDeliveryJob(job.id); + assert.deepEqual(staged?.job, job); + assert.deepEqual(staged?.bytes, bytes); + await assert.rejects( + authority.store.putImageDeliveryJob({ job, bytes: new Uint8Array([9]) }), + /immutable/, + ); + await authority.store.create({ + ...identity, + id: job.id, + name: 'image', + kind: 'image', + source: 'tool_result_projection', + content: bytes, + imageDelivery: { + messageId: identity.messageId, + source: identity.source, + status: 'ready', + contentSha256: createHash('sha256').update(bytes).digest('hex'), + }, + }); + assert.equal(await authority.store.readImageDeliveryJob(job.id), undefined); + assert.equal((await lookup(authority.store))?.status, 'ready'); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('deleting a Session also deletes pending and failed reception bytes', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-purge-')); + const authority = createSqliteArtifactStoreWriteAuthority(root); + try { + for (const failure of [undefined, 'quota_exceeded'] as const) { + await authority.store.putImageDeliveryJob({ + job: { + ...identity, + id: failure ?? 'pending-image', + loadRemote: false, + attempts: 1, + nextAttemptAt: 0, + ...(failure ? { failure } : {}), + }, + bytes: new Uint8Array([1]), + }); + } + await authority.store.purgeSessionArtifacts(identity.sessionId); + assert.equal(await authority.store.readImageDeliveryJob('pending-image'), undefined); + assert.equal(await authority.store.readImageDeliveryJob('quota_exceeded'), undefined); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('a process killed after spool commit leaves complete recoverable bytes', async () => { + const { spawnSync } = await import('node:child_process'); + const root = await mkdtemp(join(tmpdir(), 'maka-image-killed-')); + const job = { ...identity, id: 'crash-image', loadRemote: false, attempts: 1, nextAttemptAt: 0 }; + try { + const script = `import { createSqliteArtifactStoreWriteAuthority } from ${JSON.stringify(new URL('../artifact-store.js', import.meta.url).href)}; + const owner = createSqliteArtifactStoreWriteAuthority(process.argv[1]); + await owner.store.putImageDeliveryJob({ job: JSON.parse(process.argv[2]), bytes: new Uint8Array([4, 5, 6]) }); + process.kill(process.pid, 'SIGKILL');`; + const result = spawnSync( + process.execPath, + ['--input-type=module', '-e', script, root, JSON.stringify(job)], + { timeout: 10000, encoding: 'utf8' }, + ); + assert.equal(result.signal, 'SIGKILL', result.stderr); + const authority = createSqliteArtifactStoreWriteAuthority(root); + try { + const received = await authority.store.readImageDeliveryJob(job.id); + assert.deepEqual(received?.job, job); + assert.deepEqual(received?.bytes, new Uint8Array([4, 5, 6])); + } finally { + authority.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('permanent reception failures do not consume the workspace queue', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-failures-')); + const authority = createSqliteArtifactStoreWriteAuthority(root); + try { + for (let i = 0; i < 520; i++) { + await authority.store.putImageDeliveryJob({ + job: { + ...identity, + source: `/missing-${i}.png`, + id: `failed-${i}`, + loadRemote: false, + attempts: 1, + nextAttemptAt: 0, + failure: 'not_found', + }, + }); + } + const job = { ...identity, id: 'healthy', loadRemote: false, attempts: 1, nextAttemptAt: 0 }; + await authority.store.putImageDeliveryJob({ job, bytes: new Uint8Array([1]) }); + assert.equal((await authority.store.listImageDeliveryJobs(Date.now())).length, 1); + assert.equal((await authority.store.readImageDeliveryJob('healthy'))?.bytes?.length, 1); + assert.equal(await authority.store.readImageDeliveryJob('failed-0'), undefined); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +test('quota-failed spools yield capacity to healthy work without erasing failure metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-image-quota-spool-')); + const authority = createSqliteArtifactStoreWriteAuthority(root); + try { + for (let i = 0; i < 4; i++) + await authority.store.putImageDeliveryJob({ + job: { + ...identity, + source: `/quota-${i}.png`, + id: `quota-${i}`, + loadRemote: false, + attempts: 1, + nextAttemptAt: 0, + failure: 'quota_exceeded', + }, + bytes: new Uint8Array(8 * 1024 * 1024), + }); + await authority.store.putImageDeliveryJob({ + job: { ...identity, id: 'healthy', loadRemote: false, attempts: 1, nextAttemptAt: 0 }, + bytes: new Uint8Array(8 * 1024 * 1024), + }); + assert.equal(await authority.store.readImageDeliveryJob('quota-0'), undefined); + assert.deepEqual( + await authority.store.findImageDelivery( + identity.sessionId, + identity.turnId, + identity.messageId, + '/quota-0.png', + ), + { status: 'failed', reason: 'quota_exceeded' }, + ); + assert.equal( + (await authority.store.readImageDeliveryJob('healthy'))?.bytes?.byteLength, + 8 * 1024 * 1024, + ); + } finally { + authority.close(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/storage/src/artifact-store.ts b/packages/storage/src/artifact-store.ts index 745c94db5b6..e9c379f036a 100644 --- a/packages/storage/src/artifact-store.ts +++ b/packages/storage/src/artifact-store.ts @@ -17,6 +17,7 @@ * under the License. */ +import type { ImageDeliveryJob, StagedImageDeliveryJob } from './image-delivery-jobs.js'; import { assertImageArchiveQuota, planImageArchive } from './artifact-image-storage.js'; import { DEFAULT_IMAGE_ARCHIVE_LIMITS, @@ -202,6 +203,10 @@ export interface ArtifactUpgradeCleanupResult { } export interface ArtifactAuthorityStore extends DurableArtifactAttachmentReader { + putImageDeliveryJob(input: StagedImageDeliveryJob): Promise; + readImageDeliveryJob(id: string): Promise; + listImageDeliveryJobs(now: number): Promise; + deleteImageDeliveryJob(id: string): Promise; create(input: CreateArtifactInput): Promise; findImageDelivery( sessionId: string, @@ -293,6 +298,23 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { this.metadataRepository.close(); } + putImageDeliveryJob(input: StagedImageDeliveryJob): Promise { + const accepted = { + job: { ...input.job }, + ...(input.bytes ? { bytes: Uint8Array.from(input.bytes) } : {}), + }; + return this.enqueueMutation(async () => this.metadataRepository.putImageDeliveryJob(accepted)); + } + readImageDeliveryJob(id: string): Promise { + return this.enqueue(async () => this.metadataRepository.readImageDeliveryJob(id)); + } + listImageDeliveryJobs(now: number): Promise { + return this.enqueue(async () => this.metadataRepository.listImageDeliveryJobs(now)); + } + deleteImageDeliveryJob(id: string): Promise { + return this.enqueueMutation(async () => this.metadataRepository.deleteImageDeliveryJob(id)); + } + async findImageDelivery( sessionId: string, turnId: string, diff --git a/packages/storage/src/artifact-stores.ts b/packages/storage/src/artifact-stores.ts index 7f9246392ec..186cc61f853 100644 --- a/packages/storage/src/artifact-stores.ts +++ b/packages/storage/src/artifact-stores.ts @@ -48,6 +48,12 @@ export { type ReadImageSnapshotPlan, } from './artifact-attachments.js'; +export { + ImageDeliveryCapacityError, + type ImageDeliveryJob, + type StagedImageDeliveryJob, +} from './image-delivery-jobs.js'; + const writerBrand: unique symbol = Symbol('InteractiveArtifactStoreWriter'); const writers = new WeakSet(); const writerByLease = new WeakMap(); @@ -60,6 +66,10 @@ export interface InteractiveArtifactStoreWriter extends DurableArtifactAttachmen create(input: CreateArtifactInput): Promise; findImageDelivery: ArtifactAuthorityStore['findImageDelivery']; setImageDeliveryAttempt: ArtifactAuthorityStore['setImageDeliveryAttempt']; + putImageDeliveryJob: ArtifactAuthorityStore['putImageDeliveryJob']; + readImageDeliveryJob: ArtifactAuthorityStore['readImageDeliveryJob']; + listImageDeliveryJobs: ArtifactAuthorityStore['listImageDeliveryJobs']; + deleteImageDeliveryJob: ArtifactAuthorityStore['deleteImageDeliveryJob']; /** * Narrow system delete for one Session-owned artifact of a declared source. * @@ -146,6 +156,16 @@ function createWriterFacade( const acceptedAttempt = Object.freeze({ ...attempt }); return run(() => store.setImageDeliveryAttempt(acceptedIdentity, acceptedAttempt)); }, + putImageDeliveryJob: (input) => { + const accepted = { + job: { ...input.job }, + ...(input.bytes ? { bytes: Uint8Array.from(input.bytes) } : {}), + }; + return run(() => store.putImageDeliveryJob(accepted)); + }, + readImageDeliveryJob: (id) => run(() => store.readImageDeliveryJob(id)), + listImageDeliveryJobs: (now) => run(() => store.listImageDeliveryJobs(now)), + deleteImageDeliveryJob: (id) => run(() => store.deleteImageDeliveryJob(id)), listPage: (sessionId, options) => run(() => store.listPage(sessionId, options)), listTurnArtifacts: (sessionId, turnId) => run(() => store.listTurnArtifacts(sessionId, turnId)), getInSession: (sessionId, artifactId) => run(() => store.getInSession(sessionId, artifactId)), diff --git a/packages/storage/src/image-delivery-jobs.ts b/packages/storage/src/image-delivery-jobs.ts new file mode 100644 index 00000000000..c58f10b0034 --- /dev/null +++ b/packages/storage/src/image-delivery-jobs.ts @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import { + IMAGE_ARCHIVE_MAX_BYTES, + isImageDeliveryRequest, + IMAGE_DELIVERY_FAILURES, + type ImageDeliveryIdentity, + type ImageDeliveryFailure, +} from '@maka/core/image-delivery'; +import { isCanonicalArtifactEntityId } from '@maka/core/artifacts'; +import { writeImageDeliveryAttempt } from './sqlite-image-delivery.js'; + +/** Host-authorized capture intent. Payload is a bounded recovery spool, not message history. */ +export interface ImageDeliveryJob extends ImageDeliveryIdentity { + readonly id: string; + readonly publicationId?: string; + readonly publicationRunId?: string; + readonly loadRemote: boolean; + readonly attempts: number; + readonly nextAttemptAt: number; + readonly failure?: ImageDeliveryFailure; +} +export interface StagedImageDeliveryJob { + readonly job: ImageDeliveryJob; + readonly bytes?: Uint8Array; +} +export class ImageDeliveryCapacityError extends Error { + readonly name = 'ImageDeliveryCapacityError'; +} +const MAX_JOBS = 512; +const MAX_SPOOL_BYTES = 32 * 1024 * 1024; +const MAX_IMAGE_BYTES = IMAGE_ARCHIVE_MAX_BYTES; + +export function createImageDeliveryJobTable(db: DatabaseSync): void { + db.exec(`CREATE TABLE IF NOT EXISTS image_delivery_jobs ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL, + job_json TEXT NOT NULL, next_attempt_at INTEGER NOT NULL, + failed INTEGER NOT NULL CHECK (failed IN (0, 1)), payload BLOB, payload_sha256 TEXT, + CHECK (payload IS NULL OR length(payload) <= ${MAX_IMAGE_BYTES}) + ); + CREATE INDEX IF NOT EXISTS image_delivery_jobs_due + ON image_delivery_jobs(failed, next_attempt_at, id); + CREATE INDEX IF NOT EXISTS image_delivery_jobs_session ON image_delivery_jobs(session_id);`); +} + +export function putImageDeliveryJob(db: DatabaseSync, input: StagedImageDeliveryJob): void { + const { job, bytes } = input; + if ( + !isCanonicalArtifactEntityId(job.id) || + !isCanonicalArtifactEntityId(job.sessionId) || + !isImageDeliveryRequest({ turnId: job.turnId, messageId: job.messageId, source: job.source }) || + typeof job.loadRemote !== 'boolean' || + (job.failure !== undefined && !IMAGE_DELIVERY_FAILURES.includes(job.failure)) || + !Number.isSafeInteger(job.attempts) || + job.attempts < 0 || + !Number.isSafeInteger(job.nextAttemptAt) || + job.nextAttemptAt < 0 || + (job.publicationRunId !== undefined && + !isImageDeliveryRequest({ + turnId: job.publicationRunId, + messageId: job.messageId, + source: job.source, + })) || + (job.publicationId !== undefined && !isCanonicalArtifactEntityId(job.publicationId)) || + (bytes !== undefined && (!(bytes instanceof Uint8Array) || bytes.byteLength > MAX_IMAGE_BYTES)) + ) + throw new TypeError('Invalid image delivery job'); + const existing = readImageDeliveryJob(db, job.id); + if ( + existing && + [ + 'sessionId', + 'turnId', + 'messageId', + 'source', + 'publicationId', + 'publicationRunId', + 'loadRemote', + ].some( + (key) => existing.job[key as keyof ImageDeliveryJob] !== job[key as keyof ImageDeliveryJob], + ) + ) + throw new Error('Image delivery job identity conflict'); + if (existing?.bytes && bytes && !Buffer.from(existing.bytes).equals(Buffer.from(bytes))) + throw new Error('Image delivery staged bytes are immutable'); + // Terminal failures without useful bytes only need an attempt record. Invalid + // bytes must never consume the recovery spool. Quota failures retain validated + // bytes for manual retry, but are reclaimable under pressure from new work. + if (job.failure && ((!existing?.bytes && !bytes) || job.failure !== 'quota_exceeded')) { + writeImageDeliveryAttempt(db, job, { status: 'failed', reason: job.failure }); + db.prepare('DELETE FROM image_delivery_jobs WHERE id = ?').run(job.id); + return; + } + let usage = db + .prepare( + 'SELECT count(*) AS count, coalesce(sum(length(payload)), 0) AS bytes FROM image_delivery_jobs', + ) + .get() as { count: number; bytes: number }; + const exceedsCapacity = () => + (!existing && usage.count >= MAX_JOBS) || + usage.bytes - + (existing?.bytes?.byteLength ?? 0) + + (bytes?.byteLength ?? existing?.bytes?.byteLength ?? 0) > + MAX_SPOOL_BYTES; + if (exceedsCapacity()) { + // Keep failure metadata visible, while allowing healthy sessions to make + // progress. Never evict a pending job or the bytes currently being retried. + const retired = db + .prepare( + 'SELECT id, length(payload) AS bytes FROM image_delivery_jobs WHERE failed = 1 AND id != ? ORDER BY rowid', + ) + .all(job.id) as Array<{ id: string; bytes: number }>; + for (const entry of retired) { + if (!exceedsCapacity()) break; + db.prepare('DELETE FROM image_delivery_jobs WHERE id = ?').run(entry.id); + usage = { count: usage.count - 1, bytes: usage.bytes - (entry.bytes ?? 0) }; + } + } + if ( + (!existing && usage.count >= MAX_JOBS) || + usage.bytes - + (existing?.bytes?.byteLength ?? 0) + + (bytes?.byteLength ?? existing?.bytes?.byteLength ?? 0) > + MAX_SPOOL_BYTES + ) + throw new ImageDeliveryCapacityError('Image reception capacity reached'); + db.prepare(`INSERT INTO image_delivery_jobs VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET job_json = excluded.job_json, + next_attempt_at = excluded.next_attempt_at, failed = excluded.failed, + payload = coalesce(image_delivery_jobs.payload, excluded.payload), + payload_sha256 = coalesce(image_delivery_jobs.payload_sha256, excluded.payload_sha256)`).run( + job.id, + job.sessionId, + JSON.stringify(job), + job.nextAttemptAt, + job.failure ? 1 : 0, + bytes ?? null, + bytes ? createHash('sha256').update(bytes).digest('hex') : null, + ); + writeImageDeliveryAttempt( + db, + job, + job.failure ? { status: 'failed', reason: job.failure } : { status: 'pending' }, + ); +} + +export function readImageDeliveryJob( + db: DatabaseSync, + id: string, +): StagedImageDeliveryJob | undefined { + const row = db + .prepare('SELECT job_json, payload, payload_sha256 FROM image_delivery_jobs WHERE id = ?') + .get(id) as + | { job_json: string; payload: Uint8Array | null; payload_sha256: string | null } + | undefined; + if (row?.payload && createHash('sha256').update(row.payload).digest('hex') !== row.payload_sha256) + throw new Error('Image reception spool checksum mismatch'); + return row + ? { + job: JSON.parse(row.job_json) as ImageDeliveryJob, + ...(row.payload ? { bytes: row.payload } : {}), + } + : undefined; +} + +export function listImageDeliveryJobs(db: DatabaseSync, now: number): ImageDeliveryJob[] { + if (!Number.isSafeInteger(now) || now < 0) throw new TypeError('Invalid recovery time'); + return ( + db + .prepare(`SELECT job_json FROM image_delivery_jobs WHERE failed = 0 AND next_attempt_at <= ? + ORDER BY next_attempt_at, id LIMIT ${MAX_JOBS}`) + .all(now) as Array<{ job_json: string }> + ).map((row) => JSON.parse(row.job_json) as ImageDeliveryJob); +} diff --git a/packages/storage/src/sqlite-artifact-metadata.ts b/packages/storage/src/sqlite-artifact-metadata.ts index 569f9c551fe..f8d971fb213 100644 --- a/packages/storage/src/sqlite-artifact-metadata.ts +++ b/packages/storage/src/sqlite-artifact-metadata.ts @@ -28,6 +28,12 @@ import { writeImageDeliveryAttempt, removeImageDeliveryAttempt, } from './sqlite-image-delivery.js'; +import { + putImageDeliveryJob, + readImageDeliveryJob, + listImageDeliveryJobs, + type StagedImageDeliveryJob, +} from './image-delivery-jobs.js'; import type { ArtifactRecord } from '@maka/core/artifacts'; import { decodeArtifactRecordJsons } from './artifact-metadata-codec.js'; import { @@ -52,6 +58,25 @@ class SqliteArtifactMetadataRepository { this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); } + putImageDeliveryJob(input: StagedImageDeliveryJob): void { + this.assertOpen(); + this.#lease.transaction('write', () => putImageDeliveryJob(this.#lease.database, input)); + } + readImageDeliveryJob(id: string) { + this.assertOpen(); + return readImageDeliveryJob(this.#lease.database, id); + } + listImageDeliveryJobs(now: number) { + this.assertOpen(); + return listImageDeliveryJobs(this.#lease.database, now); + } + deleteImageDeliveryJob(id: string): void { + this.assertOpen(); + this.#lease.transaction('write', () => + this.#lease.database.prepare('DELETE FROM image_delivery_jobs WHERE id = ?').run(id), + ); + } + findImageDelivery( sessionId: string, turnId: string, @@ -97,6 +122,9 @@ class SqliteArtifactMetadataRepository { purgeImageDeliveryAttempts(sessionId: string): void { this.assertOpen(); this.#lease.transaction('write', () => { + this.#lease.database + .prepare('DELETE FROM image_delivery_jobs WHERE session_id = ?') + .run(sessionId); this.#lease.database .prepare('DELETE FROM image_delivery_attempts WHERE session_id = ?') .run(sessionId); @@ -142,6 +170,9 @@ class SqliteArtifactMetadataRepository { OR record_json IS NOT excluded.record_json `); for (const record of changes.upserts ?? []) { + this.#lease.database + .prepare('DELETE FROM image_delivery_jobs WHERE id = ? AND session_id = ?') + .run(record.id, record.sessionId); if (record.imageDelivery?.status === 'ready') removeImageDeliveryAttempt(this.#lease.database, { sessionId: record.sessionId, diff --git a/packages/storage/src/sqlite-artifact-schema.ts b/packages/storage/src/sqlite-artifact-schema.ts index 3ad1819785a..e622c1b9afe 100644 --- a/packages/storage/src/sqlite-artifact-schema.ts +++ b/packages/storage/src/sqlite-artifact-schema.ts @@ -23,9 +23,10 @@ import { isSafeRelativeArtifactPath, } from './artifact-metadata-codec.js'; +import { createImageDeliveryJobTable } from './image-delivery-jobs.js'; import { createImageDeliveryAttemptTable } from './sqlite-image-delivery.js'; -export const SQLITE_ARTIFACT_SCHEMA_VERSION = 4; +export const SQLITE_ARTIFACT_SCHEMA_VERSION = 5; export function migrateSqliteArtifactDatabase(db: DatabaseSync): void { const columns = db.prepare('PRAGMA table_info(artifact_records)').all() as Array<{ @@ -110,4 +111,5 @@ export function migrateSqliteArtifactDatabase(db: DatabaseSync): void { ); } createImageDeliveryAttemptTable(db); + createImageDeliveryJobTable(db); } diff --git a/packages/ui/scripts/markdown-images.browser.test.mjs b/packages/ui/scripts/markdown-images.browser.test.mjs index 4f0d873537c..8ede49226b0 100644 --- a/packages/ui/scripts/markdown-images.browser.test.mjs +++ b/packages/ui/scripts/markdown-images.browser.test.mjs @@ -361,16 +361,17 @@ test('a transient saved attachment read retries its bytes without invalidating a } finally { await page.close(); } }); -test('offscreen image performs no image request until it approaches the viewport', async () => { +test('offscreen images start Host reception but defer archived preview bytes until visible', async () => { requests = []; const page = await pageFor('offscreen'); try { - // Let React effects and the initial IntersectionObserver callback complete. + // Host reception is independent of viewport admission for rendering. await page.getByText('Loading image…').waitFor(); - assert.deepEqual(requests, []); + await page.waitForFunction(() => window.deliveryQueries === 1); assert.equal(await page.locator('img').count(), 0); await page.locator('[data-maka-image-state]').scrollIntoViewIfNeeded(); await loaded(page); + assert.equal(await page.evaluate(() => window.deliveryQueries), 1); assert.deepEqual(requests, [{ path: '/image.png', referer: undefined }]); } finally { await page.close(); } }); diff --git a/packages/ui/src/__tests__/attachment-image.test.tsx b/packages/ui/src/__tests__/attachment-image.test.tsx index 1203e69db4b..a8af93d7170 100644 --- a/packages/ui/src/__tests__/attachment-image.test.tsx +++ b/packages/ui/src/__tests__/attachment-image.test.tsx @@ -175,14 +175,6 @@ test('explains unreadable assistant attachments and offers retry', async () => { const cases: Array<[string, ReadAttachmentBytes]> = [ ['missing', async () => ({ ok: false, reason: 'not_found' })], ['document', async () => ({ ok: true, base64: 'cGRm', mimeType: 'application/pdf' })], - [ - 'large', - async () => ({ - ok: true, - base64: 'a'.repeat(3 * 1024 * 1024), - mimeType: 'image/png', - }), - ], ]; for (const [name, readBytes] of cases) { const { container } = await renderAttachmentMarkdown( @@ -271,3 +263,64 @@ test('renders an attachment when a streaming Markdown image becomes complete', a assert.ok(image); assert.equal(image.getAttribute('src'), 'data:image/png;base64,c3RyZWFt'); }); + +test('saved large images show an original-file action instead of an unsaved or failed preview', async () => { + const { container, root } = domRoot(); + const opened: string[] = []; + await act(async () => { + root.render( ({ ok: false, reason: 'too_large' })} + openSavedImage={async (sessionId, artifactId) => { opened.push(`${sessionId}/${artifactId}`); return { ok: true }; }}> + + ); + }); + assert.match(container.textContent ?? '', /Image saved.*preview size limit/); + assert.doesNotMatch(container.textContent ?? '', /not saved|loading failed/i); + assert.equal(container.querySelector('img'), null); + const button = Array.from(container.querySelectorAll('button')).find(node => node.textContent?.includes('Open saved image')); + assert.ok(button); + await act(async () => button.click()); + assert.deepEqual(opened, ['session-1/large-image']); +}); + +test('message reception status clears only when the image is durably ready', async () => { + const { ImageDeliveryProvider, ImageMessageProvider } = await import('../image-delivery.js'); + const { container, root } = domRoot(); + let finish!: (result: { status: 'ready'; artifactId: string }) => void; + await act(async () => { + root.render( ({ ok: true, mimeType: 'image/png', base64: 'c2F2ZWQ=' })}> + new Promise(resolve => { finish = resolve; })}> + + + + + ); + }); + assert.match(container.textContent ?? '', /Text has finished/); + assert.ok(container.querySelector('[data-maka-image-reception="receiving"]'), container.innerHTML); + await act(async () => finish({ status: 'ready', artifactId: 'received-image' })); + assert.equal(container.querySelector('[data-maka-image-reception]'), null); + assert.ok(container.querySelector('img')); +}); + +test('duplicate references retain reception status when one instance unmounts', async () => { + const { ImageMessageProvider, useImageReceptionStatus } = await import('../image-delivery.js'); + const { container, root } = domRoot(); + function ImageState({ state }: { state: 'receiving' | 'saved' }) { + useImageReceptionStatus('/same.png', state); + return null; + } + const render = (duplicate: boolean, messageId = 'one') => root.render( + + + {duplicate && } + , + ); + await act(() => render(true)); + assert.ok(container.querySelector('[data-maka-image-reception="receiving"]')); + await act(() => render(false)); + assert.ok(container.querySelector('[data-maka-image-reception="receiving"]')); + await act(() => render(false, 'two')); + assert.ok(container.querySelector('[data-maka-image-reception="receiving"]')); +}); diff --git a/packages/ui/src/__tests__/tool-activity-presentation.test.ts b/packages/ui/src/__tests__/tool-activity-presentation.test.ts index 5d94cad4bc0..129178daedb 100644 --- a/packages/ui/src/__tests__/tool-activity-presentation.test.ts +++ b/packages/ui/src/__tests__/tool-activity-presentation.test.ts @@ -47,6 +47,22 @@ function outputPanelCount(markup: string): number { } describe('tool activity presentation', () => { + it('keeps mixed text, image and diff previews in protocol order', () => { + const markup = renderToStaticMarkup(createElement(ToolResultPreview, { content: { + kind: 'content', parts: [ + { kind: 'text', text: 'Before the image' }, + { kind: 'image', mimeType: 'image/png', ref: { kind: 'session_file', sessionId: 'session', relativePath: 'saved-image' } }, + { kind: 'file_diff', paths: ['drawing.txt'], diff: '--- a/drawing.txt\n+++ b/drawing.txt\n@@ -1 +1 @@\n-old\n+new' }, + { kind: 'text', text: 'After the diff' }, + ], + } }), 'en'); + const before = markup.indexOf('Before the image'); + const image = markup.indexOf('maka-markdown-image-resource'); + const diff = markup.indexOf('data-kind="file_diff"'); + const after = markup.indexOf('After the diff'); + assert.ok(before >= 0 && image > before && diff > image && after > diff, markup); + }); + it('localizes client capability boundary failures and offers recovery', () => { const item: ToolActivityItem = { toolUseId: 'client-capability-boundary', diff --git a/packages/ui/src/artifact-preview-registry.ts b/packages/ui/src/artifact-preview-registry.ts index 89158dff5ca..80e8f01c0c5 100644 --- a/packages/ui/src/artifact-preview-registry.ts +++ b/packages/ui/src/artifact-preview-registry.ts @@ -63,7 +63,7 @@ function decideImagePostLoad(input: { /** Reject raw IPC payloads before base64 can enter renderer state. */ export function decideImageReadOutcome(readResult: ArtifactBinaryReadResult): ImagePostLoadOutcome { if (!readResult.ok) { - return { kind: 'unsupported', reason: 'read_failed' }; + return { kind: 'unsupported', reason: readResult.reason === 'too_large' ? 'oversize' : 'read_failed' }; } if (typeof readResult.base64 !== 'string' || typeof readResult.mimeType !== 'string') { return { kind: 'unsupported', reason: 'read_failed' }; diff --git a/packages/ui/src/attachment-image.tsx b/packages/ui/src/attachment-image.tsx index dc52a3b3311..8af4f772a6d 100644 --- a/packages/ui/src/attachment-image.tsx +++ b/packages/ui/src/attachment-image.tsx @@ -35,9 +35,13 @@ export type ReadAttachmentBytes = ( artifactId: string, ) => Promise; +type LoadedImage = { src?: string; limited?: boolean }; +export type OpenSavedImage = (sessionId: string, artifactId: string) => Promise<{ ok: boolean }>; + type SessionAttachmentContextValue = { sessionId: string; - loadImage: (sessionId: string, artifactId: string) => Promise; + openSavedImage?: OpenSavedImage; + loadImage: (sessionId: string, artifactId: string, retry?: boolean) => Promise; }; const SessionAttachmentContext = createContext(undefined); @@ -46,16 +50,18 @@ const SessionAttachmentContext = createContext { const readBytes = props.readBytes; if (!readBytes) return undefined; - const pending = new Map>(); + const pending = new Map>(); return { sessionId: props.sessionId, - loadImage(sessionId: string, artifactId: string) { + openSavedImage: props.openSavedImage, + loadImage(sessionId: string, artifactId: string, _retry = false) { const key = `${sessionId}\0${artifactId}`; const existing = pending.get(key); if (existing) return existing; @@ -63,10 +69,10 @@ export function SessionAttachmentProvider(props: { .then((result) => { const outcome = decideImageReadOutcome(result); return outcome.kind === 'image' - ? `data:${outcome.safeMime};base64,${outcome.base64}` - : undefined; + ? { src: `data:${outcome.safeMime};base64,${outcome.base64}` } + : { limited: outcome.reason === 'oversize' }; }) - .catch(() => undefined); + .catch((): LoadedImage => ({})); pending.set(key, loaded); void loaded.finally(() => { if (pending.get(key) === loaded) pending.delete(key); @@ -75,7 +81,7 @@ export function SessionAttachmentProvider(props: { }, }; }, - [props.readBytes, props.sessionId], + [props.readBytes, props.sessionId, props.openSavedImage], ); return ( @@ -112,7 +118,7 @@ export function useAttachmentImage(ref: { ); const [result, setResult] = useState<{ request: typeof request; - status: 'loading' | 'ready' | 'failed'; + status: 'loading' | 'ready' | 'failed' | 'limited'; src?: string; }>(); @@ -120,16 +126,18 @@ export function useAttachmentImage(ref: { if (!request) return; setResult({ request, status: 'loading' }); let cancelled = false; - request.loadImage(request.sessionId, request.artifactId) + request.loadImage(request.sessionId, request.artifactId, request.attempt > 0) .then((loaded) => { - if (!cancelled) setResult({ request, status: loaded ? 'ready' : 'failed', src: loaded }); + if (!cancelled) setResult({ request, status: loaded.src ? 'ready' : loaded.limited ? 'limited' : 'failed', src: loaded.src }); }); return () => { cancelled = true; }; }, [request]); - if (!request) return { status: 'unavailable' as const, src: undefined, retry }; - if (result?.request !== request) return { status: 'loading' as const, src: undefined, retry }; - return { status: result.status, src: result.src, retry }; + const openSaved = context?.openSavedImage && artifactId && sessionId + ? () => context.openSavedImage!(sessionId, artifactId) : undefined; + if (!request) return { status: 'unavailable' as const, src: undefined, retry, openSaved }; + if (result?.request !== request) return { status: 'loading' as const, src: undefined, retry, openSaved }; + return { status: result.status, src: result.src, retry, openSaved }; } diff --git a/packages/ui/src/chat-image-resource.ts b/packages/ui/src/chat-image-resource.ts index 50eb0985632..487d1ece27a 100644 --- a/packages/ui/src/chat-image-resource.ts +++ b/packages/ui/src/chat-image-resource.ts @@ -21,10 +21,11 @@ import { useState } from 'react'; import { parseAttachmentResourceRef } from '@maka/core/attachments'; import { isRemoteImageSource, type ImageDeliveryFailure } from '@maka/core/image-delivery'; import { useAttachmentImage } from './attachment-image.js'; -import { useImageDelivery } from './image-delivery.js'; +import { useImageDelivery, useImageReceptionStatus } from './image-delivery.js'; export type ChatImagePresentation = | { readonly kind: 'loading' } + | { readonly kind: 'saved_without_preview' } | { readonly kind: 'ready'; readonly src: string } | { readonly kind: 'unavailable'; readonly reason: 'attachment' | 'remote' | 'redacted' | 'unsupported' } | { readonly kind: 'failed'; readonly reason: 'saved_bytes' | ImageDeliveryFailure }; @@ -32,7 +33,7 @@ export type ChatImagePresentation = /** Owns delivery, saved-byte and decode transitions. The component only renders this view. */ export function useChatImageResource(input: { source: string; redacted: boolean; visible: boolean }) { const explicit = parseAttachmentResourceRef(input.source); - const delivery = useImageDelivery(input.source, input.visible && !explicit, !input.redacted); + const delivery = useImageDelivery(input.source, !explicit, !input.redacted); const artifactId = explicit?.artifactId ?? (delivery.status === 'ready' ? delivery.artifactId : undefined); const image = useAttachmentImage(input.visible && artifactId ? { artifactId } : undefined); const [failedSource, setFailedSource] = useState(); @@ -42,7 +43,10 @@ export function useChatImageResource(input: { source: string; redacted: boolean; const hiddenRemote = remote && input.redacted; const remotePlaceholder = input.visible && remote && !artifactId; let presentation: ChatImagePresentation; - if ((source && failedSource === source) || (artifactId && image.status === 'failed')) + useImageReceptionStatus(input.source, artifactId ? 'saved' : delivery.available + ? delivery.status === 'pending' ? 'receiving' : delivery.status === 'failed' ? 'failed' : undefined : undefined); + if (artifactId && image.status === 'limited') presentation = { kind: 'saved_without_preview' }; + else if ((source && failedSource === source) || (artifactId && image.status === 'failed')) presentation = { kind: 'failed', reason: 'saved_bytes' }; else if (source) presentation = { kind: 'ready', src: source }; else if (!input.visible) presentation = { kind: 'loading' }; @@ -64,7 +68,7 @@ export function useChatImageResource(input: { source: string; redacted: boolean; const openSource = remote && !hiddenRemote && (presentation.kind === 'failed' || presentation.kind === 'unavailable' && presentation.reason === 'remote') ? input.source : undefined; return { - presentation, attempt, retry, openSource, + presentation, attempt, retry, openSource, openSaved: image.openSaved, onDecodeError: () => { if (source) setFailedSource(source); }, }; } diff --git a/packages/ui/src/chat-view.tsx b/packages/ui/src/chat-view.tsx index 4355eb58588..add3ce8ce06 100644 --- a/packages/ui/src/chat-view.tsx +++ b/packages/ui/src/chat-view.tsx @@ -80,6 +80,7 @@ import { SessionContextLayer, type SessionContextGoal } from './session-context- import { SessionAttachmentProvider, type ReadAttachmentBytes, + type OpenSavedImage, } from './attachment-image.js'; import { MakaClientSessionScope, @@ -305,6 +306,7 @@ export function ChatView(props: { * host-agnostic with no direct host-global access. */ onReadAttachmentBytes?: ReadAttachmentBytes; + onOpenSavedImage?: OpenSavedImage; onResolveImageDelivery?: import('@maka/core/image-delivery').ResolveImageDelivery; /** * Open a linked subagent child session in the main chat column (option A). @@ -955,6 +957,7 @@ export function ChatView(props: {
void) | undefined>(undefined); +export function useImageReceptionStatus(source: string, state?: ReceptionState) { + const report = useContext(ImageReceptionScope); + const token = useId(); + useEffect(() => { report?.(token, source, state); return () => report?.(token, source); }, [report, token, source, state]); +} const ImageMessageScope = createContext<{ turnId: string; messageId: string; streaming?: boolean } | undefined>(undefined); /** Installed by assistant messages, outside the generic Markdown renderer. */ export function ImageMessageProvider(props: { @@ -31,7 +40,31 @@ export function ImageMessageProvider(props: { const value = useMemo(() => props.identity ? { turnId: props.identity.turnId, messageId: props.identity.messageId, streaming: props.streaming } : undefined, [props.identity?.turnId, props.identity?.messageId, props.streaming]); - return {props.children}; + const [states, setStates] = useState>(new Map()); + const scope = JSON.stringify([props.identity?.turnId, props.identity?.messageId]); + const report = useCallback((token: string, source: string, state?: ReceptionState) => setStates(previous => { + const current = previous.get(token); + if (current?.state === state && current?.source === source && current?.scope === scope) return previous; + const next = new Map(previous); + if (state) next.set(token, { source, state, scope }); else next.delete(token); + return next; + }), [scope]); + const copy = getSharedUiCopy(useUiLocale()).markdown; + const sources = new Map(); + for (const entry of states.values()) { + if (entry.scope !== scope) continue; + const previous = sources.get(entry.source); + if (!previous || entry.state === 'receiving' || entry.state === 'failed' && previous === 'saved') sources.set(entry.source, entry.state); + } + const receiving = [...sources.values()].filter(state => state === 'receiving').length; + const failed = [...sources.values()].filter(state => state === 'failed').length; + return + {props.children} + {(receiving > 0 || failed > 0) &&
+ {receiving > 0 && copy.imagesReceiving(receiving)} + {failed > 0 && copy.imagesFailed(failed)} +
} + ; } const DeliveryContext = createContext<{ resolve(request: ImageDeliveryRequest): Promise } | undefined>(undefined); export function ImageDeliveryProvider(props: { sessionId: string; resolve?: ResolveImageDelivery; children: ReactNode }) { diff --git a/packages/ui/src/markdown-image.tsx b/packages/ui/src/markdown-image.tsx index 5b66689ad34..59ef00081af 100644 --- a/packages/ui/src/markdown-image.tsx +++ b/packages/ui/src/markdown-image.tsx @@ -17,18 +17,13 @@ * under the License. */ -import { useEffect, useRef, useState } from 'react'; -import { Button, IconButton, Spinner, Tooltip } from '@astryxdesign/core'; import { createMarkdownPlugin, type MarkdownExtensionNode } from '@astryxdesign/core/Markdown/plugins'; import type { MarkdownAstNode, MarkdownAstRoot } from '@astryxdesign/core/Markdown'; -import { RotateCw, AlertTriangle } from './icons.js'; -import { Link } from '@astryxdesign/core/Link'; -import { useChatImageResource } from './chat-image-resource.js'; -import { getSharedUiCopy } from './shared-ui-copy.js'; -import { useUiLocale } from './locale-context.js'; import { createMarkdownImageSourceResolver } from './markdown-image-source.js'; import { redactSecrets } from './redact.js'; +import { SessionMediaImage as MarkdownImage } from './session-media-image.js'; + type PlacedImage = MarkdownExtensionNode<'maka-images', 'image', { src: string; alt: string; inline: boolean; redacted: boolean }>; // Markdown structure determines inline/block placement; decoded images supply // their natural dimensions within the presentation's CSS bounds. @@ -78,54 +73,5 @@ function isSingleImage(nodes: readonly MarkdownAstNode[]): boolean { return node.type === 'image' || (['link', 'strong', 'emphasis', 'delete'].includes(node.type) && 'children' in node && isSingleImage(node.children)); } -/** Presentation only: Host resolves local addresses and archives; UI receives artifact identities. */ -export function MarkdownImage(props: { src: string; alt: string; inline?: boolean; redacted?: boolean }) { - return ; -} -function ImageResource(props: { src: string; redacted: boolean; alt: string; inline?: boolean }) { - const copy = getSharedUiCopy(useUiLocale()).markdown; - const anchor = useRef(null); - const [visible, setVisible] = useState(typeof IntersectionObserver === 'undefined'); - useEffect(() => { - if (visible || !anchor.current) return; - const observer = new IntersectionObserver(entries => { - if (entries.some(entry => entry.isIntersecting)) { setVisible(true); observer.disconnect(); } - }, { rootMargin: '300px' }); - observer.observe(anchor.current); return () => observer.disconnect(); - }, [visible]); - const resource = useChatImageResource({ source: props.src, redacted: props.redacted, visible }); - const state = resource.presentation; - const message = state.kind === 'loading' ? copy.imageLoading - : state.kind === 'failed' ? state.reason === 'saved_bytes' ? copy.imageLoadFailed : copy.imageArchiveFailure(state.reason) - : state.kind === 'unavailable' ? { - attachment: copy.imageUnavailable, - remote: copy.imageRemoteUnavailable, - redacted: copy.imageRemoteRedacted, - unsupported: copy.imageUnsupported, - }[state.reason] : undefined; - const actions = - {resource.retry &&