From 79cedfe1fd528630418f7213d3673b2933559ebb Mon Sep 17 00:00:00 2001 From: "l.burbridge" Date: Wed, 4 Nov 2026 12:27:14 +0000 Subject: [PATCH] feat!: port plugin to the OpenCode V2 plugin API OpenCode V2 replaces the V1 plugin API, so the V1 implementation no longer loads (it failed with: Cannot find package '@opencode-ai/plugin'). - entrypoint: Plugin.define({ id, setup }) from @opencode/plugin, cleanup aborts the event subscription - events: message.* / session.error are gone; metrics now come from the step lifecycle (session.step.started, session.text.started, session.reasoning.started, session.step.streamed, session.step.ended, session.step.failed, session.execution.failed) - metrics are recorded per model step instead of per assistant message, so turns with tool calls produce one entry per request - tool: tool()/tool.schema becomes ctx.tool.transform(editor => editor.add()) with a JSON Schema input and a { content } result - toast moves to a TUI part (ctx.data.on + ctx.ui.toast.show); the server context has no TUI client - local plugin becomes a directory (.opencode/plugins/throughput/{index,tui}.ts) - split metrics/report out of the plugin for testability - tests converted from bun:test to node:test, rewritten for the step events - package.json: @opencode/plugin dependency, ./tui export, version 2.0.0 --- .gitignore | 3 +- .opencode/plugins/throughput.ts | 1 - .opencode/plugins/throughput/index.ts | 1 + .opencode/plugins/throughput/tui.ts | 1 + README.md | 64 +- package.json | 19 +- src/index.ts | 80 ++- src/metrics.ts | 169 ++++++ src/plugins/throughput.ts | 386 ------------ src/report.ts | 132 ++++ src/tui.ts | 36 ++ src/types.ts | 39 +- test/throughput.test.ts | 840 +++++++------------------- tsconfig.json | 7 +- 14 files changed, 723 insertions(+), 1055 deletions(-) delete mode 100644 .opencode/plugins/throughput.ts create mode 100644 .opencode/plugins/throughput/index.ts create mode 100644 .opencode/plugins/throughput/tui.ts create mode 100644 src/metrics.ts delete mode 100644 src/plugins/throughput.ts create mode 100644 src/report.ts create mode 100644 src/tui.ts diff --git a/.gitignore b/.gitignore index 2325c4a..f1e0f48 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ dist/ .DS_Store *.log bun.lock +package-lock.json .opencode/node_modules/ .opencode/bun.lock -.opencode/*.md \ No newline at end of file +.opencode/*.md diff --git a/.opencode/plugins/throughput.ts b/.opencode/plugins/throughput.ts deleted file mode 100644 index dc72b9b..0000000 --- a/.opencode/plugins/throughput.ts +++ /dev/null @@ -1 +0,0 @@ -export { ThroughputPlugin, default } from "../../src/plugins/throughput.js" diff --git a/.opencode/plugins/throughput/index.ts b/.opencode/plugins/throughput/index.ts new file mode 100644 index 0000000..2df1b0e --- /dev/null +++ b/.opencode/plugins/throughput/index.ts @@ -0,0 +1 @@ +export { ThroughputPlugin as default, ThroughputPlugin, PLUGIN_ID } from "../../../src/index.ts" diff --git a/.opencode/plugins/throughput/tui.ts b/.opencode/plugins/throughput/tui.ts new file mode 100644 index 0000000..75f5f79 --- /dev/null +++ b/.opencode/plugins/throughput/tui.ts @@ -0,0 +1 @@ +export { ThroughputTuiPlugin as default, ThroughputTuiPlugin } from "../../../src/tui.ts" diff --git a/README.md b/README.md index fa5b9ef..c750918 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,49 @@ # opencode-throughput -Real-time LLM performance monitoring plugin for [OpenCode](https://opencode.ai). Tracks latency, throughput, token usage, and cost per model with toast notifications and JSONL logging. +Real-time LLM performance monitoring plugin for [OpenCode](https://opencode.ai) V2. Tracks latency, throughput, token usage, and cost per model with toast notifications and JSONL logging. ## Features - **TTFT** (Time To First Token) — measures how fast the model starts generating - **TPS** (Tokens Per Second) — generation throughput during streaming -- **Total Latency** — end-to-end request time +- **Total Latency** — end-to-end time for each model step - **Token Usage** — input, output, reasoning, cache read/write - **Cost Tracking** — per-request and cumulative cost -- **Toast Notifications** — real-time performance summary after each LLM response +- **Toast Notifications** — real-time performance summary after each model step - **JSONL Logging** — persistent logs at `~/.opencode/throughput.jsonl` - **Benchmark Tool** — query historical stats on demand from the AI agent +Requires OpenCode V2 (`@opencode/plugin` 2.x). The plugin uses the V2 plugin API: +`Plugin.define({ id, setup })`, `ctx.event.subscribe`, `ctx.tool.transform`, and a +TUI part for the toast. + ## Installation -Add to your `opencode.json`: +Add to your `opencode.json(c)`: ```json { "$schema": "https://opencode.ai/config.json", - "plugin": ["opencode-throughput"] + "plugins": ["opencode-throughput"] } ``` -OpenCode will automatically install the plugin from npm at startup. +OpenCode installs the package from npm at startup and loads every part it exports: +the server part (`.`), which records metrics and registers the `benchmark` tool, and +the TUI part (`./tui`), which shows the toast. ## Usage Once installed, the plugin runs automatically: -1. **After each LLM response**, a toast notification appears with metrics: +1. **After each model step**, a toast notification appears with metrics: ``` claude-sonnet-4 | 2.1s TTFT | 58.3 tok/s | 4.2s | ↑1.2k ↓892 ``` 2. **Logs are written** to `~/.opencode/throughput.jsonl` (one JSON object per line) + and to `/.opencode/throughput.md` (one readable line per step) 3. **Query benchmarks** — ask the AI agent to use the `benchmark` tool: @@ -76,28 +83,55 @@ Each line in `~/.opencode/throughput.jsonl`: } ``` +Metrics are recorded per **step** (one model call), so a turn with tool calls +produces one entry per request. `messageID` is the assistant message the step +belongs to. + ## How It Works -The plugin hooks into OpenCode's event system: +The plugin subscribes to OpenCode V2 session events: + +| Event | Used for | +|-------|----------| +| `session.step.started` | Model reference and step start time | +| `session.text.started` / `session.reasoning.started` / `session.step.streamed` | First streamed output (TTFT) | +| `session.step.ended` | Latency, output tokens, `finish`, `cost`, token usage | +| `session.step.failed` | Failed steps, logged with `finish: "error"` | +| `session.execution.failed` | Drops steps that never completed | + +## Project layout -- `message.updated` — captures `AssistantMessage.time.completed`, `tokens`, and `cost` -- `message.part.updated` — captures the first `TextPart.time.start` for TTFT calculation +``` +src/index.ts server part: event subscription, log writers, benchmark tool +src/tui.ts TUI part: toast notifications +src/metrics.ts step tracker and formatting helpers +src/report.ts log file IO and benchmark aggregation +src/types.ts shared types +``` ## Development -### Local development +This repo is also its own local plugin. OpenCode auto-discovers +`.opencode/plugins/throughput/`, whose `index.ts` and `tui.ts` re-export the +implementation from `src/`, so edits take effect on save. + +To point another project at a checkout, configure a plugin directory (a +directory, not a file): ```json { - "plugin": ["file:///absolute/path/to/opencode-throughput/src/index.ts"] + "plugins": [{ "package": "file:///absolute/path/to/opencode-throughput" }] } ``` -### Build +Or copy the `.opencode/plugins/throughput/` directory into that project. + +### Typecheck and test ```bash -bun install -bun run build +npm install +npm run typecheck +npm test ``` ### Publish diff --git a/package.json b/package.json index 600ea46..74e6315 100644 --- a/package.json +++ b/package.json @@ -1,20 +1,24 @@ { "name": "opencode-throughput", - "version": "1.2.0", + "version": "2.0.0", "description": "Real-time LLM performance monitoring plugin for OpenCode. Tracks TTFT, TPS, latency, token usage and cost per model with toast notifications and JSONL logging.", "type": "module", "main": "./src/index.ts", "exports": { ".": "./src/index.ts", - "./plugins/*": "./src/plugins/*" + "./tui": "./src/tui.ts", + "./metrics": "./src/metrics.ts", + "./report": "./src/report.ts", + "./types": "./src/types.ts" }, "files": [ "src" ], "scripts": { - "build": "bun build src/index.ts --outdir dist --target node", - "prepublishOnly": "bun run build", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "test": "node --test --experimental-strip-types test/*.test.ts", + "build": "tsc --noEmit", + "prepublishOnly": "npm run typecheck" }, "keywords": [ "opencode", @@ -33,11 +37,10 @@ "type": "git", "url": "git+https://github.com/Howardzhangdqs/opencode-throughput.git" }, - "peerDependencies": { - "@opencode-ai/plugin": ">=1.0.0" + "dependencies": { + "@opencode/plugin": "^2.2.5" }, "devDependencies": { - "@opencode-ai/plugin": "^1.2.27", "@types/node": "^25.5.0", "typescript": "^5.7.0" } diff --git a/src/index.ts b/src/index.ts index 0200cff..699ce16 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,77 @@ -export { ThroughputPlugin } from "./plugins/throughput.js" -export { default } from "./plugins/throughput.js" -export type { LogEntry, ModelStats } from "./types.js" +import { Plugin } from "@opencode/plugin" +import { buildLogMsg, createStepTracker, TRACKED_EVENTS } from "./metrics.ts" +import { appendLog, buildBenchmarkReport, readLogs, writePerfFile } from "./report.ts" +import type { LogEntry, TrackedEvent } from "./types.ts" + +export const PLUGIN_ID = "throughput" + +const BENCHMARK_TOOL = { + name: "benchmark", + description: + "Query model performance benchmarks including latency, throughput (TTFT, TPS), token usage, and cost. Shows stats aggregated by model.", + input: { + type: "object", + properties: { + model: { + type: "string", + description: + "Filter by model name (partial match, e.g. 'claude' or 'gpt'). Omit for all models.", + }, + last: { + type: "integer", + minimum: 1, + description: "Only show the last N entries (default: show all)", + }, + }, + additionalProperties: false, + }, +} as const + +export const ThroughputPlugin = Plugin.define({ + id: PLUGIN_ID, + async setup(ctx) { + await ctx.tool.transform((editor) => { + editor.add({ + ...BENCHMARK_TOOL, + async execute(input) { + const args = (input ?? {}) as { model?: string; last?: number } + return { content: buildBenchmarkReport(readLogs(), args) } + }, + }) + }) + + const directory = ctx.location.directory + const tracker = createStepTracker() + const controller = new AbortController() + + function record(entry: LogEntry) { + try { + appendLog(entry) + } catch (error) { + console.error(`[${PLUGIN_ID}] failed to append throughput log`, error) + } + writePerfFile(directory, entry) + console.log(`[${PLUGIN_ID}] ${buildLogMsg(entry)}`) + } + + void (async () => { + try { + for await (const event of ctx.event.subscribe({ + types: [...TRACKED_EVENTS], + signal: controller.signal, + })) { + const entry = tracker.handle(event as TrackedEvent) + if (entry) record(entry) + } + } catch (error) { + if (!controller.signal.aborted) { + console.error(`[${PLUGIN_ID}] event stream failed`, error) + } + } + })() + + return () => controller.abort() + }, +}) + +export default ThroughputPlugin diff --git a/src/metrics.ts b/src/metrics.ts new file mode 100644 index 0000000..ed15882 --- /dev/null +++ b/src/metrics.ts @@ -0,0 +1,169 @@ +import type { LogEntry, TrackedEvent } from "./types.ts" + +/** Events that carry timing information for one model step. */ +export const TRACKED_EVENTS = [ + "session.step.started", + "session.step.streamed", + "session.step.ended", + "session.step.failed", + "session.text.started", + "session.reasoning.started", + "session.execution.failed", +] as const + +export type TrackedEventType = (typeof TRACKED_EVENTS)[number] + +/** Drop half-finished steps that never produced an end event. */ +const STALE_STEP_MS = 15 * 60 * 1000 + +type PendingStep = { + sessionID: string + providerID: string + modelID: string + started: number + firstOutput?: number +} + +export function modelKey(providerID: string, modelID: string): string { + return `${providerID}/${modelID}` +} + +/** + * Turns OpenCode V2 step events into one {@link LogEntry} per model step. + * + * A step starts with `session.step.started`, the first streamed token or + * reasoning chunk marks time-to-first-token, and `session.step.ended` (or + * `session.step.failed`) closes it with token and cost accounting. + */ +export function createStepTracker(now: () => number = Date.now) { + const pending = new Map() + + function prune() { + const cutoff = now() - STALE_STEP_MS + for (const [id, step] of pending) { + if (step.started < cutoff) pending.delete(id) + } + } + + function start(event: TrackedEvent) { + const data = event.data ?? {} + const id = data.assistantMessageID + if (!id) return + pending.set(id, { + sessionID: data.sessionID ?? "", + providerID: data.model?.providerID ?? "", + modelID: data.model?.id ?? "", + started: data.started ?? event.created, + }) + } + + function firstOutput(event: TrackedEvent) { + const id = event.data?.assistantMessageID + if (!id) return + const step = pending.get(id) + if (!step || step.firstOutput !== undefined) return + step.firstOutput = event.created + } + + function finish(event: TrackedEvent): LogEntry | undefined { + const data = event.data ?? {} + const id = data.assistantMessageID + if (!id) return + + const step = pending.get(id) + // Without a start event there is nothing to measure against, and no model + // to attribute the step to. This only happens when the plugin loads while a + // step is already streaming. + if (!step) return + pending.delete(id) + + const ended = event.created + const latency = ended - step.started + const ttft = step.firstOutput === undefined ? null : step.firstOutput - step.started + const tokens = data.tokens ?? {} + const output = tokens.output ?? 0 + const generation = step.firstOutput === undefined ? 0 : ended - step.firstOutput + const tps = generation > 0 && output > 0 ? (output / generation) * 1000 : null + const finish = data.finish ?? (event.type === "session.step.failed" ? "error" : undefined) + + return { + ts: new Date(event.created).toISOString(), + model: modelKey(step.providerID, step.modelID), + providerID: step.providerID, + modelID: step.modelID, + sessionID: data.sessionID ?? step.sessionID, + messageID: id, + ttft_ms: ttft, + tps, + latency_ms: latency, + inputTokens: tokens.input ?? 0, + outputTokens: output, + reasoningTokens: tokens.reasoning ?? 0, + cacheReadTokens: tokens.cache?.read ?? 0, + cacheWriteTokens: tokens.cache?.write ?? 0, + cost: data.cost ?? 0, + finish, + } + } + + function handle(event: TrackedEvent): LogEntry | undefined { + prune() + switch (event.type) { + case "session.step.started": + start(event) + return + case "session.text.started": + case "session.reasoning.started": + case "session.step.streamed": + firstOutput(event) + return + case "session.step.ended": + case "session.step.failed": + return finish(event) + case "session.execution.failed": { + // The run aborted: drop anything still pending for that session. + const sessionID = event.data?.sessionID + if (!sessionID) return + for (const [id, step] of pending) { + if (step.sessionID === sessionID) pending.delete(id) + } + return + } + default: + return + } + } + + return { handle, pending } +} + +export function formatNum(n: number | null): string { + if (n == null) return "N/A" + if (n >= 1000000) return (n / 1000000).toFixed(2) + "M" + if (n >= 1000) return (n / 1000).toFixed(1) + "k" + return n.toFixed(n >= 100 ? 0 : 1) +} + +export function formatMs(ms: number | null): string { + if (ms == null) return "N/A" + if (ms >= 60000) return (ms / 60000).toFixed(1) + "m" + if (ms >= 1000) return (ms / 1000).toFixed(1) + "s" + return ms.toFixed(0) + "ms" +} + +export function buildLogMsg(entry: LogEntry): string { + const shortModel = + entry.modelID.length > 28 ? entry.modelID.slice(0, 26) + ".." : entry.modelID + const parts = [ + shortModel, + formatMs(entry.ttft_ms) + " TTFT", + formatNum(entry.tps) + " tok/s", + formatMs(entry.latency_ms), + "\u2191" + formatNum(entry.inputTokens) + " \u2193" + formatNum(entry.outputTokens), + ] + return parts.join(" | ") +} + +export function toastVariant(entry: LogEntry): "info" | "success" | "warning" | "error" { + return entry.finish === "error" ? "warning" : "info" +} diff --git a/src/plugins/throughput.ts b/src/plugins/throughput.ts deleted file mode 100644 index c57e3e2..0000000 --- a/src/plugins/throughput.ts +++ /dev/null @@ -1,386 +0,0 @@ -import type { Plugin } from "@opencode-ai/plugin" -import { tool } from "@opencode-ai/plugin" -import type { LogEntry, ModelStats } from "../types.js" -import path from "path" -import fs from "fs" -import os from "os" - -const LOG_DIR = path.join(os.homedir(), ".opencode") -const LOG_FILE = path.join(LOG_DIR, "throughput.jsonl") - -function ensureLogDir() { - if (!fs.existsSync(LOG_DIR)) { - fs.mkdirSync(LOG_DIR, { recursive: true }) - } -} - -function getModelKey(providerID: string, modelID: string): string { - return `${providerID}/${modelID}` -} - -function updateStats(stats: Map, key: string, entry: LogEntry) { - const existing = stats.get(key) - if (!existing) { - stats.set(key, { - model: key, - count: 1, - avgTTFT: entry.ttft_ms ?? 0, - avgTPS: entry.tps ?? 0, - avgLatency: entry.latency_ms ?? 0, - minTTFT: entry.ttft_ms ?? Infinity, - maxTTFT: entry.ttft_ms ?? 0, - minLatency: entry.latency_ms ?? Infinity, - maxLatency: entry.latency_ms ?? 0, - totalInputTokens: entry.inputTokens, - totalOutputTokens: entry.outputTokens, - totalReasoningTokens: entry.reasoningTokens, - totalCacheReadTokens: entry.cacheReadTokens, - totalCacheWriteTokens: entry.cacheWriteTokens, - totalCost: entry.cost, - lastUpdated: Date.now(), - }) - return - } - - const c = existing.count + 1 - existing.count = c - existing.avgTTFT = (existing.avgTTFT * (c - 1) + (entry.ttft_ms ?? 0)) / c - existing.avgTPS = (existing.avgTPS * (c - 1) + (entry.tps ?? 0)) / c - existing.avgLatency = (existing.avgLatency * (c - 1) + (entry.latency_ms ?? 0)) / c - if (entry.ttft_ms != null) { - existing.minTTFT = Math.min(existing.minTTFT, entry.ttft_ms) - existing.maxTTFT = Math.max(existing.maxTTFT, entry.ttft_ms) - } - if (entry.latency_ms != null) { - existing.minLatency = Math.min(existing.minLatency, entry.latency_ms) - existing.maxLatency = Math.max(existing.maxLatency, entry.latency_ms) - } - existing.totalInputTokens += entry.inputTokens - existing.totalOutputTokens += entry.outputTokens - existing.totalReasoningTokens += entry.reasoningTokens - existing.totalCacheReadTokens += entry.cacheReadTokens - existing.totalCacheWriteTokens += entry.cacheWriteTokens - existing.totalCost += entry.cost - existing.lastUpdated = Date.now() - stats.set(key, existing) -} - -function formatNum(n: number | null): string { - if (n == null) return "N/A" - if (n >= 1000000) return (n / 1000000).toFixed(2) + "M" - if (n >= 1000) return (n / 1000).toFixed(1) + "k" - return n.toFixed(n >= 100 ? 0 : 1) -} - -function formatMs(ms: number | null): string { - if (ms == null) return "N/A" - if (ms >= 60000) return (ms / 60000).toFixed(1) + "m" - if (ms >= 1000) return (ms / 1000).toFixed(1) + "s" - return ms.toFixed(0) + "ms" -} - -function appendLog(entry: LogEntry) { - ensureLogDir() - const line = JSON.stringify(entry) + "\n" - fs.appendFileSync(LOG_FILE, line, "utf-8") -} - -function buildLogMsg(entry: LogEntry): string { - const shortModel = - entry.modelID.length > 28 ? entry.modelID.slice(0, 26) + ".." : entry.modelID - const parts = [ - shortModel, - formatMs(entry.ttft_ms) + " TTFT", - formatNum(entry.tps) + " tok/s", - formatMs(entry.latency_ms), - "\u2191" + formatNum(entry.inputTokens) + " \u2193" + formatNum(entry.outputTokens), - ] - return parts.join(" | ") -} - -function readLogs(): LogEntry[] { - if (!fs.existsSync(LOG_FILE)) return [] - const content = fs.readFileSync(LOG_FILE, "utf-8") - return content - .split("\n") - .filter((l: string) => l.trim()) - .map((l: string) => JSON.parse(l) as LogEntry) -} - -const benchmarkTool = tool({ - description: - "Query model performance benchmarks including latency, throughput (TTFT, TPS), token usage, and cost. Shows stats aggregated by model.", - args: { - model: tool.schema - .string() - .optional() - .describe( - "Filter by model name (partial match, e.g. 'claude' or 'gpt'). Omit for all models." - ), - last: tool.schema - .number() - .optional() - .describe("Only show the last N entries (default: show all)"), - }, - async execute(args) { - const logs = readLogs() - - if (logs.length === 0) { - return "No benchmark data yet. The throughput plugin needs to process at least one LLM response before stats are available." - } - - const keyword = args.model?.toLowerCase() - const filtered = keyword - ? logs.filter( - (l: LogEntry) => - l.model.toLowerCase().includes(keyword) || - l.modelID.toLowerCase().includes(keyword) - ) - : logs - - const sliced = args.last ? filtered.slice(-args.last) : filtered - - const byModel = new Map() - - for (const entry of sliced) { - const arr = byModel.get(entry.model) ?? [] - arr.push(entry) - byModel.set(entry.model, arr) - } - - const lines: string[] = [] - lines.push(`## Model Performance Benchmark`) - lines.push(`Total requests: ${sliced.length}`) - if (args.model) lines.push(`Filter: ${args.model}`) - lines.push(``) - - for (const [model, entries] of byModel) { - const count = entries.length - const ttfts = entries - .filter((e: LogEntry) => e.ttft_ms != null) - .map((e: LogEntry) => e.ttft_ms!) - const latencies = entries - .filter((e: LogEntry) => e.latency_ms != null) - .map((e: LogEntry) => e.latency_ms!) - const tpsList = entries - .filter((e: LogEntry) => e.tps != null) - .map((e: LogEntry) => e.tps!) - - const avgTTFT = - ttfts.length > 0 ? ttfts.reduce((a: number, b: number) => a + b, 0) / ttfts.length : null - const minTTFT = ttfts.length > 0 ? Math.min(...ttfts) : null - const maxTTFT = ttfts.length > 0 ? Math.max(...ttfts) : null - - const avgLatency = - latencies.length > 0 - ? latencies.reduce((a: number, b: number) => a + b, 0) / latencies.length - : null - const minLatency = latencies.length > 0 ? Math.min(...latencies) : null - const maxLatency = latencies.length > 0 ? Math.max(...latencies) : null - - const avgTPS = - tpsList.length > 0 - ? tpsList.reduce((a: number, b: number) => a + b, 0) / tpsList.length - : null - const maxTPS = tpsList.length > 0 ? Math.max(...tpsList) : null - - const totalInput = entries.reduce((s: number, e: LogEntry) => s + e.inputTokens, 0) - const totalOutput = entries.reduce((s: number, e: LogEntry) => s + e.outputTokens, 0) - const totalReasoning = entries.reduce( - (s: number, e: LogEntry) => s + e.reasoningTokens, - 0 - ) - const totalCacheRead = entries.reduce( - (s: number, e: LogEntry) => s + e.cacheReadTokens, - 0 - ) - const totalCacheWrite = entries.reduce( - (s: number, e: LogEntry) => s + e.cacheWriteTokens, - 0 - ) - const totalCost = entries.reduce((s: number, e: LogEntry) => s + e.cost, 0) - - lines.push(`### ${model}`) - lines.push(` Requests: ${count}`) - lines.push(``) - lines.push(` **Latency:**`) - lines.push( - ` TTFT: avg ${formatMs(avgTTFT)} | min ${formatMs(minTTFT)} | max ${formatMs(maxTTFT)}` - ) - lines.push( - ` Total: avg ${formatMs(avgLatency)} | min ${formatMs(minLatency)} | max ${formatMs(maxLatency)}` - ) - lines.push(``) - lines.push(` **Throughput:**`) - lines.push(` TPS: avg ${formatNum(avgTPS)} | max ${formatNum(maxTPS)}`) - lines.push(``) - lines.push(` **Tokens:**`) - lines.push(` Input: ${formatNum(totalInput)}`) - lines.push(` Output: ${formatNum(totalOutput)}`) - lines.push(` Reasoning: ${formatNum(totalReasoning)}`) - lines.push(` Cache R/W: ${formatNum(totalCacheRead)} / ${formatNum(totalCacheWrite)}`) - lines.push(``) - lines.push(` **Cost:** $${totalCost.toFixed(4)}`) - - if (entries.length <= 5) { - lines.push(``) - lines.push(` Recent entries:`) - for (const e of entries) { - lines.push( - ` ${e.ts} | ${formatMs(e.ttft_ms)} TTFT | ${formatNum(e.tps)} tok/s | ${formatMs(e.latency_ms)} | cost $${e.cost.toFixed(4)}` - ) - } - } - - lines.push(``) - } - - return lines.join("\n") - }, -}) - -export const ThroughputPlugin: Plugin = async ({ client, directory }) => { - const stats = new Map() - const firstPartTime = new Map() - const msgCreatedTime = new Map() - - function writePerfFile(entry: LogEntry) { - if (!directory) return - const throughputFile = path.join(directory, ".opencode", "throughput.md") - - const line = `${entry.ts} | ${entry.modelID} | TTFT ${formatMs(entry.ttft_ms)} | TPS ${formatNum(entry.tps)} tok/s | Latency ${formatMs(entry.latency_ms)} | ↑${formatNum(entry.inputTokens)} ↓${formatNum(entry.outputTokens)} ↓r${formatNum(entry.reasoningTokens)} | Cost $${entry.cost.toFixed(4)}${entry.finish && entry.finish !== "stop" ? " | " + entry.finish : ""}\n` - - try { - fs.mkdirSync(path.dirname(throughputFile), { recursive: true }) - fs.appendFileSync(throughputFile, line, "utf-8") - } catch {} - } - - return { - tool: { - benchmark: benchmarkTool, - }, - - event: async ({ event }) => { - if (event.type === "message.updated") { - const info = event.properties.info as any - if (info?.role !== "assistant") return - - const msgID = info.id as string - - if (info.time?.created && !msgCreatedTime.has(msgID)) { - msgCreatedTime.set(msgID, info.time.created) - } - - if (info.time?.completed) { - const created = info.time.created as number - const completed = info.time.completed as number - - if (!created || !completed) { - firstPartTime.delete(msgID) - msgCreatedTime.delete(msgID) - return - } - - const latencyMs = completed - created - const firstPart = firstPartTime.get(msgID) - const ttftMs = firstPart ? firstPart - created : null - const outputTokens = (info.tokens?.output as number) ?? 0 - const genTimeMs = firstPart ? completed - firstPart : latencyMs - const tps = genTimeMs > 0 && outputTokens > 0 ? (outputTokens / genTimeMs) * 1000 : null - - const providerID = (info.providerID as string) ?? "" - const modelID = (info.modelID as string) ?? "" - const modelKey = getModelKey(providerID, modelID) - - const entry: LogEntry = { - ts: new Date().toISOString(), - model: modelKey, - providerID, - modelID, - sessionID: info.sessionID as string, - messageID: msgID, - ttft_ms: ttftMs, - tps, - latency_ms: latencyMs, - inputTokens: (info.tokens?.input as number) ?? 0, - outputTokens, - reasoningTokens: (info.tokens?.reasoning as number) ?? 0, - cacheReadTokens: (info.tokens?.cache?.read as number) ?? 0, - cacheWriteTokens: (info.tokens?.cache?.write as number) ?? 0, - cost: (info.cost as number) ?? 0, - finish: info.finish as string | undefined, - } - - updateStats(stats, modelKey, entry) - appendLog(entry) - writePerfFile(entry) - - const logMsg = buildLogMsg(entry) - const variant = entry.finish === "error" ? "warn" : "info" - const toastVariant: "info" | "success" | "warning" | "error" = - entry.finish === "error" ? "warning" : "info" - - try { - await client.tui.showToast({ - body: { - title: "Throughput", - message: logMsg, - variant: toastVariant, - duration: 4000, - }, - }) - } catch {} - - try { - await client.app.log({ - body: { - service: "opencode-throughput", - level: variant, - message: logMsg, - extra: { - model: entry.model, - ttft_ms: entry.ttft_ms, - tps: entry.tps, - latency_ms: entry.latency_ms, - inputTokens: entry.inputTokens, - outputTokens: entry.outputTokens, - cost: entry.cost, - }, - }, - }) - } catch {} - - firstPartTime.delete(msgID) - msgCreatedTime.delete(msgID) - } - } - - if (event.type === "message.removed") { - const props = event.properties as any - const msgID = props?.info?.id ?? props?.messageID - if (msgID) { - firstPartTime.delete(msgID as string) - msgCreatedTime.delete(msgID as string) - } - } - - if (event.type === "session.error") { - firstPartTime.clear() - msgCreatedTime.clear() - } - - if (event.type === "message.part.updated") { - const part = (event.properties as any).part as any - if (!part) return - - const msgID = part.messageID as string - if (part.type === "text" && part.time?.start && !firstPartTime.has(msgID)) { - firstPartTime.set(msgID, part.time.start) - } - } - }, - } -} - -export default ThroughputPlugin diff --git a/src/report.ts b/src/report.ts new file mode 100644 index 0000000..21937ae --- /dev/null +++ b/src/report.ts @@ -0,0 +1,132 @@ +import path from "node:path" +import fs from "node:fs" +import os from "node:os" +import type { LogEntry } from "./types.ts" +import { formatMs, formatNum } from "./metrics.ts" + +export const LOG_DIR = path.join(os.homedir(), ".opencode") +export const LOG_FILE = path.join(LOG_DIR, "throughput.jsonl") + +function ensureLogDir() { + if (!fs.existsSync(LOG_DIR)) { + fs.mkdirSync(LOG_DIR, { recursive: true }) + } +} + +export function appendLog(entry: LogEntry) { + ensureLogDir() + fs.appendFileSync(LOG_FILE, JSON.stringify(entry) + "\n", "utf-8") +} + +export function readLogs(): LogEntry[] { + if (!fs.existsSync(LOG_FILE)) return [] + const content = fs.readFileSync(LOG_FILE, "utf-8") + return content + .split("\n") + .filter((l: string) => l.trim()) + .map((l: string) => JSON.parse(l) as LogEntry) +} + +/** Appends a human-readable one-liner to `/.opencode/throughput.md`. */ +export function writePerfFile(directory: string | undefined, entry: LogEntry) { + if (!directory) return + const throughputFile = path.join(directory, ".opencode", "throughput.md") + + const line = `${entry.ts} | ${entry.modelID} | TTFT ${formatMs(entry.ttft_ms)} | TPS ${formatNum(entry.tps)} tok/s | Latency ${formatMs(entry.latency_ms)} | ↑${formatNum(entry.inputTokens)} ↓${formatNum(entry.outputTokens)} ↓r${formatNum(entry.reasoningTokens)} | Cost $${entry.cost.toFixed(4)}${entry.finish && entry.finish !== "stop" ? " | " + entry.finish : ""}\n` + + try { + fs.mkdirSync(path.dirname(throughputFile), { recursive: true }) + fs.appendFileSync(throughputFile, line, "utf-8") + } catch {} +} + +function average(values: number[]): number | null { + if (values.length === 0) return null + return values.reduce((a, b) => a + b, 0) / values.length +} + +function sum(entries: LogEntry[], pick: (entry: LogEntry) => number): number { + return entries.reduce((total, entry) => total + pick(entry), 0) +} + +export type BenchmarkQuery = { + model?: string + last?: number +} + +export function buildBenchmarkReport( + logs: LogEntry[], + query: BenchmarkQuery = {} +): string { + if (logs.length === 0) { + return "No benchmark data yet. The throughput plugin needs to process at least one LLM response before stats are available." + } + + const keyword = query.model?.toLowerCase() + const filtered = keyword + ? logs.filter( + (l) => l.model.toLowerCase().includes(keyword) || l.modelID.toLowerCase().includes(keyword) + ) + : logs + + const sliced = query.last ? filtered.slice(-query.last) : filtered + + const byModel = new Map() + for (const entry of sliced) { + const arr = byModel.get(entry.model) ?? [] + arr.push(entry) + byModel.set(entry.model, arr) + } + + const lines: string[] = [] + lines.push(`## Model Performance Benchmark`) + lines.push(`Total requests: ${sliced.length}`) + if (query.model) lines.push(`Filter: ${query.model}`) + lines.push(``) + + for (const [model, entries] of byModel) { + const ttfts = entries.filter((e) => e.ttft_ms != null).map((e) => e.ttft_ms!) + const latencies = entries.filter((e) => e.latency_ms != null).map((e) => e.latency_ms!) + const tpsList = entries.filter((e) => e.tps != null).map((e) => e.tps!) + + lines.push(`### ${model}`) + lines.push(` Requests: ${entries.length}`) + lines.push(``) + lines.push(` **Latency:**`) + lines.push( + ` TTFT: avg ${formatMs(average(ttfts))} | min ${formatMs(ttfts.length ? Math.min(...ttfts) : null)} | max ${formatMs(ttfts.length ? Math.max(...ttfts) : null)}` + ) + lines.push( + ` Total: avg ${formatMs(average(latencies))} | min ${formatMs(latencies.length ? Math.min(...latencies) : null)} | max ${formatMs(latencies.length ? Math.max(...latencies) : null)}` + ) + lines.push(``) + lines.push(` **Throughput:**`) + lines.push( + ` TPS: avg ${formatNum(average(tpsList))} | max ${formatNum(tpsList.length ? Math.max(...tpsList) : null)}` + ) + lines.push(``) + lines.push(` **Tokens:**`) + lines.push(` Input: ${formatNum(sum(entries, (e) => e.inputTokens))}`) + lines.push(` Output: ${formatNum(sum(entries, (e) => e.outputTokens))}`) + lines.push(` Reasoning: ${formatNum(sum(entries, (e) => e.reasoningTokens))}`) + lines.push( + ` Cache R/W: ${formatNum(sum(entries, (e) => e.cacheReadTokens))} / ${formatNum(sum(entries, (e) => e.cacheWriteTokens))}` + ) + lines.push(``) + lines.push(` **Cost:** $${sum(entries, (e) => e.cost).toFixed(4)}`) + + if (entries.length <= 5) { + lines.push(``) + lines.push(` Recent entries:`) + for (const e of entries) { + lines.push( + ` ${e.ts} | ${formatMs(e.ttft_ms)} TTFT | ${formatNum(e.tps)} tok/s | ${formatMs(e.latency_ms)} | cost $${e.cost.toFixed(4)}` + ) + } + } + + lines.push(``) + } + + return lines.join("\n") +} diff --git a/src/tui.ts b/src/tui.ts new file mode 100644 index 0000000..629be1f --- /dev/null +++ b/src/tui.ts @@ -0,0 +1,36 @@ +import { Plugin } from "@opencode/plugin/tui" +import { buildLogMsg, createStepTracker, toastVariant, TRACKED_EVENTS } from "./metrics.ts" + +/** + * TUI part of the throughput plugin: renders the per-step metrics as a toast in + * the terminal client. The server part keeps the logs and the `benchmark` tool. + */ +export const ThroughputTuiPlugin = Plugin.define({ + id: "throughput.tui", + setup(ctx) { + const tracker = createStepTracker() + const unsubscribes: Array<() => void> = [] + + for (const type of TRACKED_EVENTS) { + unsubscribes.push( + ctx.data.on(type, (event) => { + const entry = tracker.handle(event) + if (!entry) return + ctx.ui.toast.show({ + title: "Throughput", + message: buildLogMsg(entry), + variant: toastVariant(entry), + duration: 4000, + sessionID: entry.sessionID, + }) + }) + ) + } + + return () => { + for (const unsubscribe of unsubscribes) unsubscribe() + } + }, +}) + +export default ThroughputTuiPlugin diff --git a/src/types.ts b/src/types.ts index c4edd4c..792d4f9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,21 +17,26 @@ export type LogEntry = { finish: string | undefined } -export type ModelStats = { - model: string - count: number - avgTTFT: number - avgTPS: number - avgLatency: number - minTTFT: number - maxTTFT: number - minLatency: number - maxLatency: number - totalInputTokens: number - totalOutputTokens: number - totalReasoningTokens: number - totalCacheReadTokens: number - totalCacheWriteTokens: number - totalCost: number - lastUpdated: number +/** + * The OpenCode V2 events the plugin consumes. Only the fields that are read are + * declared, so both the server event stream and the TUI data store satisfy it. + */ +export type TrackedEvent = { + type: string + created: number + data?: { + sessionID?: string + assistantMessageID?: string + model?: { id?: string; providerID?: string } + started?: number + finish?: string + cost?: number + tokens?: { + input?: number + output?: number + reasoning?: number + cache?: { read?: number; write?: number } + } + error?: { message?: string } + } } diff --git a/test/throughput.test.ts b/test/throughput.test.ts index f1e7f4b..cbb8a41 100644 --- a/test/throughput.test.ts +++ b/test/throughput.test.ts @@ -1,679 +1,279 @@ -import { describe, test, expect, beforeEach, afterEach } from "bun:test" -import fs from "fs" -import os from "os" -import path from "path" -import type { LogEntry } from "../src/types.js" - -const TEST_DIR = path.join(os.tmpdir(), "opencode-throughput-test-" + process.pid) -const REAL_LOG_DIR = path.join(os.homedir(), ".opencode") -const REAL_LOG_FILE = path.join(REAL_LOG_DIR, "throughput.jsonl") - -const logCalls: Array<{ service: string; level: string; message: string; extra: any }> = [] -let baseLogCount = 0 - -const mockClient = { - app: { - log: async (opts: any) => { - logCalls.push(opts.body) - return true - }, +import { describe, test, beforeEach } from "node:test" +import assert from "node:assert/strict" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { buildLogMsg, createStepTracker, formatMs, formatNum, toastVariant } from "../src/metrics.ts" +import { buildBenchmarkReport, writePerfFile } from "../src/report.ts" +import type { LogEntry, TrackedEvent } from "../src/types.ts" + +/** Fixed clock so the tracker's stale-step pruning never interferes with fixtures. */ +const BASE = 1_700_000_000_000 +const clock = () => BASE + 60_000 + +const stepStarted = (overrides: Partial = {}): TrackedEvent => ({ + type: "session.step.started", + created: BASE + 1_000, + data: { + sessionID: "sess-1", + assistantMessageID: "msg-1", + model: { id: "claude-sonnet-4", providerID: "anthropic" }, + started: BASE + 1_000, + ...overrides, }, -} +}) -function getLogContent(): LogEntry[] { - if (!fs.existsSync(REAL_LOG_FILE)) return [] - return fs - .readFileSync(REAL_LOG_FILE, "utf-8") - .split("\n") - .filter((l: string) => l.trim()) - .map((l: string) => JSON.parse(l) as LogEntry) -} +const streamed = (type = "session.text.started", created = BASE + 3_000): TrackedEvent => ({ + type, + created, + data: { sessionID: "sess-1", assistantMessageID: "msg-1" }, +}) -function removeLastLineFromLog() { - if (!fs.existsSync(REAL_LOG_FILE)) return - const content = fs.readFileSync(REAL_LOG_FILE, "utf-8") - const lines = content.split("\n").filter((l: string) => l.trim()) - if (lines.length === 0) { - fs.unlinkSync(REAL_LOG_FILE) - } else { - lines.pop() - fs.writeFileSync(REAL_LOG_FILE, lines.join("\n") + "\n", "utf-8") - } -} +const stepEnded = (overrides: Partial = {}, created = BASE + 6_000): TrackedEvent => ({ + type: "session.step.ended", + created, + data: { + sessionID: "sess-1", + assistantMessageID: "msg-1", + finish: "stop", + cost: 0.01, + tokens: { input: 1000, output: 300, reasoning: 0, cache: { read: 200, write: 100 } }, + ...overrides, + }, +}) -function makeAssistantMessage(overrides: Record = {}) { +function entry(overrides: Partial = {}): LogEntry { return { - id: "msg-1", - sessionID: "sess-1", - role: "assistant", - providerID: "anthropic", - modelID: "claude-sonnet-4", - time: { created: Date.now() - 5000, completed: Date.now() }, - tokens: { input: 1000, output: 500, reasoning: 0, cache: { read: 200, write: 100 } }, + ts: "2026-03-21T10:00:00Z", + model: "a/b", + providerID: "a", + modelID: "b", + sessionID: "s1", + messageID: "m1", + ttft_ms: 2000, + tps: 50, + latency_ms: 5000, + inputTokens: 1000, + outputTokens: 500, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, cost: 0.01, finish: "stop", ...overrides, } } -async function createPlugin() { - const { ThroughputPlugin } = await import("../src/plugins/throughput.js") - const hooks = await ThroughputPlugin({ client: mockClient } as any) - return hooks -} - -function snapshotNewEntries(): LogEntry[] { - const all = getLogContent() - return all.slice(baseLogCount) -} - -function cleanupNewEntries() { - const all = getLogContent() - const toRemove = all.length - baseLogCount - for (let i = 0; i < toRemove; i++) { - removeLastLineFromLog() - } -} - -beforeEach(() => { - logCalls.length = 0 - baseLogCount = getLogContent().length -}) - -afterEach(() => { - cleanupNewEntries() -}) - -describe("event handling: complete flow", () => { - test("part.updated -> message.updated: TTFT, TPS, latency, tokens, cost all recorded", async () => { - const hooks = await createPlugin() - const createdTime = Date.now() - 5000 - const completedTime = Date.now() - const partStartTime = createdTime + 2000 - - await hooks.event!({ - event: { - type: "message.part.updated", - properties: { - part: { - type: "text", - messageID: "msg-1", - time: { start: partStartTime }, - }, - }, - } as any, - }) - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: createdTime, completed: completedTime }, - }), - }, - } as any, - }) - - expect(logCalls.length).toBe(1) - expect(logCalls[0].service).toBe("opencode-throughput") - expect(logCalls[0].level).toBe("info") - expect(logCalls[0].message).toContain("claude-sonnet-4") - expect(logCalls[0].message).toContain("TTFT") - expect(logCalls[0].message).toContain("tok/s") - expect(logCalls[0].extra.ttft_ms).toBe(2000) - expect(logCalls[0].extra.tps).toBeGreaterThan(0) - expect(logCalls[0].extra.latency_ms).toBe(5000) - expect(logCalls[0].extra.inputTokens).toBe(1000) - expect(logCalls[0].extra.outputTokens).toBe(500) - expect(logCalls[0].extra.cost).toBe(0.01) - - const logs = snapshotNewEntries() - expect(logs.length).toBe(1) - expect(logs[0].ttft_ms).toBe(2000) - expect(logs[0].latency_ms).toBe(5000) - expect(logs[0].tps).toBeGreaterThan(0) - expect(logs[0].model).toBe("anthropic/claude-sonnet-4") - expect(logs[0].inputTokens).toBe(1000) - expect(logs[0].outputTokens).toBe(500) - expect(logs[0].reasoningTokens).toBe(0) - expect(logs[0].cacheReadTokens).toBe(200) - expect(logs[0].cacheWriteTokens).toBe(100) - expect(logs[0].cost).toBe(0.01) - expect(logs[0].finish).toBe("stop") - expect(logs[0].sessionID).toBe("sess-1") - }) - - test("no TTFT when message.part.updated is missing", async () => { - const hooks = await createPlugin() - const createdTime = Date.now() - 5000 - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: createdTime, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.ttft_ms).toBeNull() - const logs = snapshotNewEntries() - expect(logs[0].ttft_ms).toBeNull() - }) - - test("no TTFT when part is not text type", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.part.updated", - properties: { - part: { - type: "tool_use", - messageID: "msg-1", - time: { start: Date.now() - 3000 }, - }, - }, - } as any, - }) - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.ttft_ms).toBeNull() - }) - - test("no TTFT when part has no time.start", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.part.updated", - properties: { - part: { - type: "text", - messageID: "msg-1", - }, - }, - } as any, - }) - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.ttft_ms).toBeNull() +describe("step tracker", () => { + test("started -> first token -> ended produces all metrics", () => { + const tracker = createStepTracker(clock) + assert.equal(tracker.handle(stepStarted()), undefined) + assert.equal(tracker.handle(streamed()), undefined) + + const log = tracker.handle(stepEnded()) + assert.ok(log) + assert.equal(log.ttft_ms, 2000) + assert.equal(log.latency_ms, 5000) + assert.equal(log.tps, 100) // 300 tokens over 3000ms + assert.equal(log.model, "anthropic/claude-sonnet-4") + assert.equal(log.providerID, "anthropic") + assert.equal(log.modelID, "claude-sonnet-4") + assert.equal(log.sessionID, "sess-1") + assert.equal(log.messageID, "msg-1") + assert.equal(log.inputTokens, 1000) + assert.equal(log.outputTokens, 300) + assert.equal(log.cacheReadTokens, 200) + assert.equal(log.cacheWriteTokens, 100) + assert.equal(log.cost, 0.01) + assert.equal(log.finish, "stop") }) -}) - -describe("edge cases", () => { - test("message without time.created AND time.completed: skip entirely", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls.length).toBe(0) - expect(snapshotNewEntries().length).toBe(0) - }) - - test("message without time.completed: no trigger", async () => { - const hooks = await createPlugin() - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls.length).toBe(0) + test("reasoning chunks also mark the first output", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + tracker.handle(streamed("session.reasoning.started", BASE + 2_500)) + const log = tracker.handle(stepEnded()) + assert.equal(log?.ttft_ms, 1500) }) - test("user role messages are ignored", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: { - role: "user", - id: "msg-1", - time: { created: Date.now() - 5000, completed: Date.now() }, - }, - }, - } as any, - }) - - expect(logCalls.length).toBe(0) + test("no first output: TTFT and TPS are null, latency still measured", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + const log = tracker.handle(stepEnded()) + assert.ok(log) + assert.equal(log.ttft_ms, null) + assert.equal(log.tps, null) + assert.equal(log.latency_ms, 5000) }) - test("error finish sets warn level", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - finish: "error", - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].level).toBe("warn") + test("zero output tokens: TPS is null", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + tracker.handle(streamed()) + const log = tracker.handle( + stepEnded({ tokens: { input: 1000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } } }) + ) + assert.equal(log?.tps, null) }) - test("zero output tokens: TPS is null", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - tokens: { input: 1000, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.tps).toBeNull() + test("end event without a start event is ignored", () => { + const tracker = createStepTracker(clock) + assert.equal(tracker.handle(stepEnded()), undefined) }) - test("missing tokens field: defaults to 0", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - tokens: undefined, - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.inputTokens).toBe(0) - expect(logCalls[0].extra.outputTokens).toBe(0) - const logs = snapshotNewEntries() - expect(logs[0].inputTokens).toBe(0) + test("missing token and cost fields default to zero", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + const log = tracker.handle(stepEnded({ tokens: undefined, cost: undefined })) + assert.equal(log?.inputTokens, 0) + assert.equal(log?.outputTokens, 0) + assert.equal(log?.reasoningTokens, 0) + assert.equal(log?.cost, 0) }) - test("missing providerID/modelID: defaults to empty string", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - providerID: undefined, - modelID: undefined, - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, + test("failed step is logged with an error finish", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + const log = tracker.handle({ + type: "session.step.failed", + created: BASE + 4_000, + data: { + sessionID: "sess-1", + assistantMessageID: "msg-1", + cost: 0.002, + tokens: { input: 10, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + }, }) - - expect(logCalls[0].extra.model).toBe("/") - const logs = snapshotNewEntries() - expect(logs[0].model).toBe("/") + assert.ok(log) + assert.equal(log.finish, "error") + assert.equal(log.latency_ms, 3000) + assert.equal(toastVariant(log), "warning") }) - test("message.removed cleans up pending TTFT tracking", async () => { - const hooks = await createPlugin() - const createdTime = Date.now() - 5000 - - await hooks.event!({ - event: { - type: "message.part.updated", - properties: { - part: { - type: "text", - messageID: "msg-1", - time: { start: createdTime + 2000 }, - }, - }, - } as any, - }) - - await hooks.event!({ - event: { - type: "message.removed", - properties: { messageID: "msg-1" }, - } as any, + test("failed execution clears pending steps", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + tracker.handle({ + type: "session.execution.failed", + created: BASE + 2_000, + data: { sessionID: "sess-1" }, }) - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - id: "msg-1", - time: { created: createdTime, completed: Date.now() }, - }), - }, - } as any, - }) - - expect(logCalls[0].extra.ttft_ms).toBeNull() + assert.equal(tracker.pending.size, 0) + assert.equal(tracker.handle(stepEnded()), undefined) }) - test("message.part.updated without part property: no crash", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.part.updated", - properties: {}, - } as any, - }) - - expect(logCalls.length).toBe(0) + test("abandoned steps are pruned after 15 minutes", () => { + let now = BASE + 1_000 + const tracker = createStepTracker(() => now) + tracker.handle(stepStarted()) + assert.equal(tracker.pending.size, 1) + now = BASE + 1_000 + 16 * 60 * 1000 + tracker.handle({ type: "unrelated", created: now }) + assert.equal(tracker.pending.size, 0) }) - test("message.updated without properties.info: no crash", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: {}, - } as any, - }) - - expect(logCalls.length).toBe(0) + test("concurrent steps from different messages stay separate", () => { + const tracker = createStepTracker(clock) + tracker.handle(stepStarted()) + tracker.handle(stepStarted({ assistantMessageID: "msg-2", model: { id: "gpt-5", providerID: "openai" } })) + tracker.handle(streamed()) + + const first = tracker.handle(stepEnded()) + const second = tracker.handle(stepEnded({ assistantMessageID: "msg-2" }, BASE + 8_000)) + assert.equal(first?.model, "anthropic/claude-sonnet-4") + assert.equal(second?.model, "openai/gpt-5") + assert.equal(second?.latency_ms, 7000) }) }) -describe("multi-message aggregation", () => { - test("two messages from same model accumulate in log", async () => { - const hooks = await createPlugin() - - for (let i = 0; i < 2; i++) { - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - id: `msg-${i}`, - time: { created: Date.now() - 5000, completed: Date.now() }, - cost: 0.01, - }), - }, - } as any, - }) - } - - expect(logCalls.length).toBe(2) - expect(snapshotNewEntries().length).toBe(2) - const totalCost = snapshotNewEntries().reduce((s, l) => s + l.cost, 0) - expect(totalCost).toBeCloseTo(0.02, 5) +describe("formatting", () => { + test("formatMs", () => { + assert.equal(formatMs(null), "N/A") + assert.equal(formatMs(250), "250ms") + assert.equal(formatMs(2500), "2.5s") + assert.equal(formatMs(90_000), "1.5m") }) - test("messages from different models both logged", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - id: "msg-1", - providerID: "anthropic", - modelID: "claude-sonnet-4", - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - id: "msg-2", - providerID: "openai", - modelID: "gpt-5", - cost: 0.02, - time: { created: Date.now() - 3000, completed: Date.now() }, - }), - }, - } as any, - }) - - const logs = snapshotNewEntries() - expect(logs.length).toBe(2) - expect(logs[0].model).toBe("anthropic/claude-sonnet-4") - expect(logs[1].model).toBe("openai/gpt-5") + test("formatNum", () => { + assert.equal(formatNum(null), "N/A") + assert.equal(formatNum(5), "5.0") + assert.equal(formatNum(250), "250") + assert.equal(formatNum(1500), "1.5k") + assert.equal(formatNum(2_500_000), "2.50M") }) -}) -describe("log message format", () => { - test("contains model name, TTFT, TPS, latency, token counts", async () => { - const hooks = await createPlugin() - - await hooks.event!({ - event: { - type: "message.updated", - properties: { - info: makeAssistantMessage({ - time: { created: Date.now() - 5000, completed: Date.now() }, - }), - }, - } as any, - }) - - const msg = logCalls[0].message - expect(msg).toContain("claude-sonnet-4") - expect(msg).toContain("TTFT") - expect(msg).toContain("tok/s") - expect(msg).toContain("\u2191") - expect(msg).toContain("\u2193") + test("log message contains model, TTFT, TPS, latency and tokens", () => { + const msg = buildLogMsg(entry()) + assert.match(msg, /b \| 2\.0s TTFT \| 50\.0 tok\/s \| 5\.0s \| ↑1\.0k ↓500/) }) }) -describe("benchmark tool", () => { - let backupContent: string | null = null - const LOG_FILE = path.join(os.homedir(), ".opencode", "throughput.jsonl") - - beforeEach(() => { - if (fs.existsSync(LOG_FILE)) { - backupContent = fs.readFileSync(LOG_FILE, "utf-8") - fs.unlinkSync(LOG_FILE) - } +describe("benchmark report", () => { + test("empty log returns a helpful message", () => { + assert.match(buildBenchmarkReport([]), /No benchmark data yet/) }) - afterEach(() => { - if (fs.existsSync(LOG_FILE)) fs.unlinkSync(LOG_FILE) - if (backupContent !== null) { - fs.writeFileSync(LOG_FILE, backupContent, "utf-8") - backupContent = null - } + test("single entry shows model stats", () => { + const report = buildBenchmarkReport([entry()]) + assert.match(report, /anthropic\/claude-sonnet-4|a\/b/) + assert.match(report, /Requests: 1/) + assert.match(report, /avg 2\.0s/) + assert.match(report, /\$0\.0100/) + assert.match(report, /Cache R\/W/) }) - async function createPluginWithBenchmark() { - const { ThroughputPlugin } = await import("../src/plugins/throughput.js") - const hooks = await ThroughputPlugin({ client: mockClient } as any) - return hooks - } - - function writeTestLogs(entries: LogEntry[]) { - fs.mkdirSync(path.dirname(LOG_FILE), { recursive: true }) - const lines = entries.map((e) => JSON.stringify(e)).join("\n") + "\n" - fs.writeFileSync(LOG_FILE, lines, "utf-8") - } + test("filters by model name, case-insensitively", () => { + const report = buildBenchmarkReport( + [entry(), entry({ model: "openai/gpt-5", modelID: "gpt-5" })], + { model: "GPT" } + ) + assert.match(report, /openai\/gpt-5/) + assert.doesNotMatch(report, /a\/b/) + }) - test("no data: returns helpful message", async () => { - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({}, {} as any) - expect(result).toContain("No benchmark data yet") + test("last limits the sample", () => { + const report = buildBenchmarkReport( + [entry({ ttft_ms: 1000, tps: 10, latency_ms: 1000 }), entry({ ttft_ms: 2000, tps: 20, latency_ms: 2000 })], + { last: 1 } + ) + assert.match(report, /Total requests: 1/) }) - test("single entry: shows model stats", async () => { - writeTestLogs([ - { - ts: "2026-03-21T10:00:00Z", - model: "anthropic/claude-sonnet-4", - providerID: "anthropic", - modelID: "claude-sonnet-4", - sessionID: "s1", - messageID: "m1", - ttft_ms: 2000, - tps: 50, - latency_ms: 5000, - inputTokens: 1000, - outputTokens: 500, - reasoningTokens: 0, - cacheReadTokens: 200, - cacheWriteTokens: 100, - cost: 0.01, - finish: "stop", - }, + test("aggregates across entries", () => { + const report = buildBenchmarkReport([ + entry({ ttft_ms: 2000, tps: 50, latency_ms: 5000, cost: 0.01 }), + entry({ ttft_ms: 4000, tps: 100, latency_ms: 8000, cost: 0.02 }), ]) - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({}, {} as any) as string - expect(result).toContain("anthropic/claude-sonnet-4") - expect(result).toContain("Requests: 1") - expect(result).toContain("2.0s") - expect(result).toContain("50") - expect(result).toContain("$0.0100") - expect(result).toContain("Cache R/W") + assert.match(report, /Total requests: 2/) + assert.match(report, /avg 3\.0s/) + assert.match(report, /min 2\.0s/) + assert.match(report, /max 4\.0s/) + assert.match(report, /\$0\.0300/) }) +}) - test("filter by model name", async () => { - writeTestLogs([ - { - ts: "2026-03-21T10:00:00Z", - model: "anthropic/claude-sonnet-4", - providerID: "anthropic", - modelID: "claude-sonnet-4", - sessionID: "s1", - messageID: "m1", - ttft_ms: 2000, tps: 50, latency_ms: 5000, - inputTokens: 1000, outputTokens: 500, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.01, finish: "stop", - }, - { - ts: "2026-03-21T10:01:00Z", - model: "openai/gpt-5", - providerID: "openai", - modelID: "gpt-5", - sessionID: "s1", - messageID: "m2", - ttft_ms: 500, tps: 100, latency_ms: 3000, - inputTokens: 800, outputTokens: 300, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.02, finish: "stop", - }, - ]) - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({ model: "gpt" }, {} as any) as string - expect(result).toContain("openai/gpt-5") - expect(result).not.toContain("claude-sonnet-4") +describe("perf markdown file", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "throughput-md-")) + + beforeEach(() => { + fs.rmSync(path.join(dir, ".opencode"), { recursive: true, force: true }) }) - test("case-insensitive filter", async () => { - writeTestLogs([ - { - ts: "2026-03-21T10:00:00Z", - model: "anthropic/claude-sonnet-4", - providerID: "anthropic", - modelID: "claude-sonnet-4", - sessionID: "s1", - messageID: "m1", - ttft_ms: 2000, tps: 50, latency_ms: 5000, - inputTokens: 1000, outputTokens: 500, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.01, finish: "stop", - }, - ]) - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({ model: "CLAUDE" }, {} as any) as string - expect(result).toContain("claude-sonnet-4") + test("appends a line under .opencode/throughput.md", () => { + writePerfFile(dir, entry()) + const file = path.join(dir, ".opencode", "throughput.md") + assert.ok(fs.existsSync(file)) + const content = fs.readFileSync(file, "utf-8") + assert.match(content, /TTFT 2\.0s \| TPS 50\.0 tok\/s/) }) - test("last parameter limits entries", async () => { - writeTestLogs([ - { - ts: "2026-03-21T10:00:00Z", model: "a/b", providerID: "a", modelID: "b", - sessionID: "s1", messageID: "m1", - ttft_ms: 1000, tps: 10, latency_ms: 1000, - inputTokens: 100, outputTokens: 50, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.001, finish: "stop", - }, - { - ts: "2026-03-21T10:01:00Z", model: "a/b", providerID: "a", modelID: "b", - sessionID: "s1", messageID: "m2", - ttft_ms: 2000, tps: 20, latency_ms: 2000, - inputTokens: 200, outputTokens: 100, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.002, finish: "stop", - }, - ]) - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({ last: 1 }, {} as any) as string - expect(result).toContain("Total requests: 1") + test("no directory is a no-op", () => { + assert.doesNotThrow(() => writePerfFile(undefined, entry())) }) - test("aggregation across multiple entries", async () => { - writeTestLogs([ - { - ts: "2026-03-21T10:00:00Z", model: "a/b", providerID: "a", modelID: "b", - sessionID: "s1", messageID: "m1", - ttft_ms: 2000, tps: 50, latency_ms: 5000, - inputTokens: 1000, outputTokens: 500, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.01, finish: "stop", - }, - { - ts: "2026-03-21T10:01:00Z", model: "a/b", providerID: "a", modelID: "b", - sessionID: "s1", messageID: "m2", - ttft_ms: 4000, tps: 100, latency_ms: 8000, - inputTokens: 2000, outputTokens: 1000, reasoningTokens: 0, - cacheReadTokens: 0, cacheWriteTokens: 0, cost: 0.02, finish: "stop", - }, - ]) - const hooks = await createPluginWithBenchmark() - const result = await hooks.tool!.benchmark.execute({}, {} as any) as string - expect(result).toContain("Total requests: 2") - expect(result).toContain("avg 3.0s") - expect(result).toContain("min 2.0s") - expect(result).toContain("max 4.0s") - expect(result).toContain("$0.0300") + test("notes a non-stop finish", () => { + writePerfFile(dir, entry({ finish: "length" })) + const content = fs.readFileSync(path.join(dir, ".opencode", "throughput.md"), "utf-8") + assert.match(content, /\| length\n$/) }) }) diff --git a/tsconfig.json b/tsconfig.json index fd820ef..9371616 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,12 +3,11 @@ "target": "ES2022", "module": "ESNext", "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "noEmit": true, "esModuleInterop": true, "strict": true, - "skipLibCheck": true, - "declaration": true, - "outDir": "./dist", - "rootDir": "./src" + "skipLibCheck": true }, "include": ["src/**/*.ts"], "exclude": ["node_modules", "dist"]