From 16aaaf71b47fc629c80418409dbadba2b07caddb Mon Sep 17 00:00:00 2001 From: Martin B Date: Fri, 23 Oct 2026 13:25:29 +0200 Subject: [PATCH 1/3] fix(usage): retain Kiro prompt estimates in hosted tool paths --- .../src/content/docs/reference/adapters.md | 4 +- scripts/test-layout/layout.json | 1 + src/server/responses/sidecar-execution.ts | 2 + structure/transports/responses.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- .../responses/responses-sidecar-usage.test.ts | 120 ++++++++++++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 tests/responses/responses-sidecar-usage.test.ts diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 435e83f1a90..cec43afb836 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -491,7 +491,9 @@ MiMo model Command Code serves. The plan's precise credit balance is retained with that identity-fenced quota reading. Separately, Kiro `meteringEvent` credit values are measured request spend: the last value within a physical response is used, and credits from separately billed sends are added. - Token usage remains estimated; credits are never inferred from tokens. + Token usage remains estimated; credits are never inferred from tokens. Requests with hosted + search or image tools retain the adapter's full prompt estimate in Usage and daily activity, + including replayed history, even when the model does not invoke those tools. - After an admitted request, reads that account's available models from the regional management service without delaying the request. Cached results add model IDs to the shipped catalog. Empty or unrecognised replies retain the shipped list and any last good account list. Model diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2fdded5a374..50deb08bdc2 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1546,6 +1546,7 @@ "responses-console-go-upload-retry.test.ts": "responses", "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", + "responses-sidecar-usage.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", "responses-core-modules.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses", diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index c9fbea9a167..aafa06617ff 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -584,6 +584,7 @@ export async function executeResponsesSidecars( return fetch.unpacedFetch ?? fetch; }, onRequestBuilt: request => { + if (typeof request.usageLog?.inputTokens === "number") logCtx.usageLogInputTokens = request.usageLog.inputTokens; recordAdapterReasoning(logCtx, request); recordAdapterTier(logCtx, request); }, @@ -694,6 +695,7 @@ export async function executeResponsesSidecars( abortSignal: options.abortSignal, onFirstOutput: noteSidecarOutput, onRequestBuilt: request => { + if (typeof request.usageLog?.inputTokens === "number") logCtx.usageLogInputTokens = request.usageLog.inputTokens; recordAdapterReasoning(logCtx, request); recordAdapterTier(logCtx, request); }, diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 02f6eb0a8b6..d76aa8dd317 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -426,7 +426,7 @@ The continuation-note and continuation-wire tests establish prompt and wire cont | `passthrough-dispatch.ts` | Native request preparation, upstream sends and pre-commit recovery. | | `passthrough-delivery.ts`, `terminal-error-redaction.ts`, `non-replayable-error.ts` | Native HTTP/SSE/JSON delivery and terminal accounting. Marked real errors use bounded reads to retain only allowlisted error type and code, keep status and non-replayability plus `x-should-retry: false` when present, and withhold upstream body text and all other upstream headers; the client receives a fixed generic message and synthetic refusals stay distinct. SSE diagnostics and xAI tool-envelope filtering precede client delivery/continuation storage. | | `policy-refusal.ts` | Rewrites an allowlisted non-combo HTTP 403 model refusal (`isUpstreamPolicyRefusal` in `src/lib/errors.ts`) from an xAI destination only (`isXaiResponsesDestination`: api.x.ai or the Grok CLI proxy, on either wire) to an HTTP 200 Responses `incomplete` / `content_filter` payload, JSON or SSE, for both `adapter-dispatch.ts` and `passthrough-delivery.ts`. A streamed rewrite takes the turn admission lease and releases it when the body finishes, so the refusal stays inside active-turn accounting. Combo attempts keep the original 403 so failover classifies it as a hop. | -| `sidecar-execution.ts`, `sidecar-send-budget.ts` | Image/video versus web-search execution, shared rotation, and adapter-aware prepaid inference/producer ownership under the [spend contract](responses-spend.md#prepaid-initial-sends). | +| `sidecar-execution.ts`, `sidecar-send-budget.ts` | Image/video versus web-search execution, per-build adapter prompt estimates for usage logging, shared rotation, and adapter-aware prepaid inference/producer ownership under the [spend contract](responses-spend.md#prepaid-initial-sends). | | `completion-policy.ts`, `run-turn-execution.ts` | Empty-completion eligibility and adapter-owned event turns; refreshed retries reject previously failed key bindings and revalidate after each queued admission; adapter-owned CLI spawns receive the post-setup `IncomingMeta.beforePhysicalSend` fence, with [replacement-key readmission](responses-failover.md#upstream-key-attempt-accounting). | | `adapter-dispatch.ts` | Translated initial dispatch, bounded recovery (including [Anthropic thinking-signature recovery](anthropic-thinking-recovery.md)) and the shared continuation retry counter. Initial and rebuilt adapter budget refusals retain the HTTP 413 [byte-accounting contract](byte-accounting.md#request-copy-accounting); ordinary build errors remain 400. | | `adapter-continuation.ts`, `adapter-delivery.ts` | Continuation event sources and final streaming/buffered bridging; a streamed turn with a `clientEncoder` option is handed to `src/server/inference/client-encoder-delivery.ts` instead of the bridge. | diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 32e61e52725..b0bbd5911f7 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1105,7 +1105,7 @@ "responses-compaction-policy-identity.test.ts": "responses", "responses-compaction-http-isolation.test.ts": "responses", "responses-compaction-routing.test.ts": "responses", "native-compaction-gateway.test.ts": "responses", "native-compaction-gateway-policy.test.ts": "responses", "native-compaction-provenance.test.ts": "responses", "native-compaction-oauth.test.ts": "responses", "native-compaction-gateway-credentials.test.ts": "responses", "responses-compaction.test.ts": "responses", "responses-console-go-upload-retry.test.ts": "responses", - "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", + "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", "responses-sidecar-usage.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", "responses-core-modules.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses", "responses-custom-tool-historical-replay.test.ts": "responses", "responses-custom-tool-repair-dispatch.test.ts": "responses", "responses-custom-tool-repair.test.ts": "responses", diff --git a/tests/responses/responses-sidecar-usage.test.ts b/tests/responses/responses-sidecar-usage.test.ts new file mode 100644 index 00000000000..420426a4477 --- /dev/null +++ b/tests/responses/responses-sidecar-usage.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { KIRO_COMPLETION_TOOL_NAME } from "../../src/adapters/kiro-constants"; +import { encodeMessage } from "../../src/lib/eventstream-decoder"; +import { saveCredential } from "../../src/oauth/store"; +import { handleResponses } from "../../src/server/responses/core"; +import { responseWithDeferredRequestLog } from "../../src/server/relay"; +import type { RequestLogContext } from "../../src/server/request-log"; +import { readUsageEntries, resetUsageReadCacheForTests } from "../../src/usage/log"; +import { summarizeUsage } from "../../src/usage/summary"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +const originalHome = process.env.OPENCODEX_HOME; +let home: string; +let isolated: IsolatedCodexHome; +let release: (() => void) | undefined; + +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-sidecar-usage-")); + isolated = installIsolatedCodexHome("ocx-sidecar-usage-codex-"); + process.env.OPENCODEX_HOME = home; + release = acquireOwnedSpendHome(); + resetUsageReadCacheForTests(); + await saveCredential("kiro", { + access: "synthetic-kiro-access", refresh: "synthetic-kiro-refresh", + expires: Date.now() + 3_600_000, source: "oauth", + kiro: { apiRegion: "us-east-1" }, + }); +}); + +afterEach(() => { + release?.(); + release = undefined; + globalThis.fetch = originalFetch; + resetUsageReadCacheForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + isolated.restore(); + removeTreeWithRetry(home); +}); + +function completedAnswer(): Response { + const frame = (type: string, value: Record) => encodeMessage( + { ":message-type": "event", ":event-type": type }, + new TextEncoder().encode(JSON.stringify(value)), + ); + const tool = { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "synthetic-completion" }; + return new Response(Buffer.concat([ + frame("toolUseEvent", tool), + frame("toolUseEvent", { ...tool, input: JSON.stringify({ answer: "Completed fixture answer." }) }), + frame("toolUseEvent", { ...tool, stop: true }), + ]), { headers: { "content-type": "application/vnd.amazon.eventstream" } }); +} + +describe("sidecar prompt usage accounting", () => { + for (const sidecar of ["none", "web-search", "images"] as const) { + test(`${sidecar}: persists the full Kiro prompt estimate into daily activity`, async () => { + let sends = 0; + globalThis.fetch = (async input => { + const url = input instanceof Request ? input.url : String(input); + if (!/^https:\/\/runtime\.[a-z0-9-]+\.kiro\.dev\/$/.test(url)) { + throw new Error("Unexpected network target blocked by sidecar usage fixture"); + } + sends++; + return completedAnswer(); + }) as typeof fetch; + const config = { + defaultProvider: "kiro", emptyCompletionRetry: false, + webSearchSidecar: { backend: "exa", exaApiKey: "synthetic-search-key" }, + images: { bridgeEnabled: true }, + providers: { + kiro: { adapter: "kiro", baseUrl: "https://runtime.us-east-1.kiro.dev", + authMode: "oauth", models: ["claude-opus-5.5"] }, + xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", + authMode: "key", apiKey: "synthetic-image-key", models: ["fixture"] }, + }, + } as OcxConfig; + const request = new Request("http://127.0.0.1/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "kiro/claude-opus-5.5", stream: true, store: false, + input: [ + { role: "user", content: "Earlier fixture context. ".repeat(4096) }, + { role: "assistant", content: "Earlier fixture answer." }, + { role: "user", content: "Continue." }, + ], + tools: [ + { type: "function", name: "fixture_lookup", parameters: { type: "object", properties: {} } }, + ...(sidecar === "none" ? [] : [{ type: sidecar === "web-search" ? "web_search" : "image_generation" }]), + ], + }), + }); + const log: RequestLogContext = { model: "", provider: "" }; + const started = Date.now(); + const response = await handleResponses(request, config, log); + const logged = responseWithDeferredRequestLog(response, `sidecar-usage-${sidecar}`, started, log); + expect(response.status).toBe(200); + expect(await logged.text()).toContain("Completed fixture answer."); + expect(sends).toBe(1); + + const entries = readUsageEntries(); + expect(entries).toHaveLength(1); + expect(entries[0]!.usageStatus).toBe("estimated"); + // The newest user message is tiny; the persisted request and physical attempt must + // still include the replayed prompt, just as the ordinary dispatch path does. + expect(entries[0]!.usage!.inputTokens).toBeGreaterThan(20_000); + expect(entries[0]!.attempts![0]!.usage!.inputTokens).toBeGreaterThan(20_000); + const summary = summarizeUsage(entries, "today", Date.now()); + expect(summary.summary.requests).toBe(1); + expect(summary.summary.totalTokens).toBe(entries[0]!.totalTokens!); + expect(summary.days[0]!.totalTokens).toBeGreaterThan(20_000); + expect(summary.models[0]!.inputTokens).toBeGreaterThan(20_000); + }); + } +}); From d294de2799360ce9139c86ce19aa210f0a5a53d8 Mon Sep 17 00:00:00 2001 From: Martin B Date: Fri, 23 Oct 2026 16:01:17 +0200 Subject: [PATCH 2/3] test(usage): document synthetic Kiro completion fixture --- tests/responses/responses-sidecar-usage.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/responses/responses-sidecar-usage.test.ts b/tests/responses/responses-sidecar-usage.test.ts index 420426a4477..157ac20a0e2 100644 --- a/tests/responses/responses-sidecar-usage.test.ts +++ b/tests/responses/responses-sidecar-usage.test.ts @@ -45,6 +45,7 @@ afterEach(() => { removeTreeWithRetry(home); }); +/** Build the synthetic Kiro completion frames consumed by all three dispatch fixtures. */ function completedAnswer(): Response { const frame = (type: string, value: Record) => encodeMessage( { ":message-type": "event", ":event-type": type }, From 3d4c28417745736ada723bdf75c03f96db2e644e Mon Sep 17 00:00:00 2001 From: Martin B Date: Sat, 24 Oct 2026 15:43:39 +0200 Subject: [PATCH 3/3] test(usage): register sidecar usage coverage in an explicit shard --- scripts/test-layout/layout-usage.json | 3 +++ scripts/test-layout/layout.json | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 scripts/test-layout/layout-usage.json diff --git a/scripts/test-layout/layout-usage.json b/scripts/test-layout/layout-usage.json new file mode 100644 index 00000000000..5e5f2d85fb0 --- /dev/null +++ b/scripts/test-layout/layout-usage.json @@ -0,0 +1,3 @@ +{ + "responses-sidecar-usage.test.ts": "responses" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 50deb08bdc2..435ab863ffd 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-usage.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", @@ -1546,7 +1546,6 @@ "responses-console-go-upload-retry.test.ts": "responses", "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", - "responses-sidecar-usage.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", "responses-core-modules.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses",