From dcf99e5bacbf59b9ff08583c3ecfd9b796d070b5 Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:44:52 +0900 Subject: [PATCH 1/2] fix(usage): retain Kiro prompt estimates in hosted tool paths Carry of #7620 onto current dev. The hosted web-search and image/video sidecar paths now record the adapter's usageLog.inputTokens estimate in onRequestBuilt, matching ordinary adapter dispatch. Co-authored-by: Martin Becerra <197874+martinbcg@users.noreply.github.com> --- .../src/content/docs/reference/adapters.md | 4 +- scripts/test-layout/layout-usage.json | 3 + scripts/test-layout/layout.json | 2 +- src/server/responses/sidecar-execution.ts | 2 + structure/transports/responses.md | 2 +- tests/fixtures/test-layout-expected.json | 2 +- .../responses/responses-sidecar-usage.test.ts | 121 ++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 scripts/test-layout/layout-usage.json create mode 100644 tests/responses/responses-sidecar-usage.test.ts diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index b4eb39b789e..b81796182f1 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -497,7 +497,9 @@ MiMo model Command Code serves. The plan's precise credit balance is retained with that identity-fenced quota reading. Separately, Kiro `meteringEvent` credit values are measured request spend: the last value within a physical response is used, and credits from separately billed sends are added. - Token usage remains estimated; credits are never inferred from tokens. + Token usage remains estimated; credits are never inferred from tokens. Requests with hosted + search or image tools retain the adapter's full prompt estimate in Usage and daily activity, + including replayed history, even when the model does not invoke those tools. - After an admitted request, reads that account's available models from the regional management service without delaying the request. Cached results add model IDs to the shipped catalog. Empty or unrecognised replies retain the shipped list and any last good account list. Model diff --git a/scripts/test-layout/layout-usage.json b/scripts/test-layout/layout-usage.json new file mode 100644 index 00000000000..5e5f2d85fb0 --- /dev/null +++ b/scripts/test-layout/layout-usage.json @@ -0,0 +1,3 @@ +{ + "responses-sidecar-usage.test.ts": "responses" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index f3a4bcc5fd8..b8d73e0cc93 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-usage.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", diff --git a/src/server/responses/sidecar-execution.ts b/src/server/responses/sidecar-execution.ts index e474c4bcd5b..932e87c6080 100644 --- a/src/server/responses/sidecar-execution.ts +++ b/src/server/responses/sidecar-execution.ts @@ -585,6 +585,7 @@ export async function executeResponsesSidecars( return fetch.unpacedFetch ?? fetch; }, onRequestBuilt: request => { + if (typeof request.usageLog?.inputTokens === "number") logCtx.usageLogInputTokens = request.usageLog.inputTokens; recordAdapterReasoning(logCtx, request); recordAdapterTier(logCtx, request); }, @@ -698,6 +699,7 @@ export async function executeResponsesSidecars( abortSignal: options.abortSignal, onFirstOutput: noteSidecarOutput, onRequestBuilt: request => { + if (typeof request.usageLog?.inputTokens === "number") logCtx.usageLogInputTokens = request.usageLog.inputTokens; recordAdapterReasoning(logCtx, request); recordAdapterTier(logCtx, request); }, diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 6275ba66101..ce089db05bb 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -426,7 +426,7 @@ The continuation-note and continuation-wire tests establish prompt and wire cont | `passthrough-dispatch.ts` | Native request preparation, upstream sends and pre-commit recovery. | | `passthrough-delivery.ts`, `terminal-error-redaction.ts`, `non-replayable-error.ts` | Native HTTP/SSE/JSON delivery and terminal accounting. Marked real errors use bounded reads to retain only allowlisted error type and code, keep status and non-replayability plus `x-should-retry: false` when present, and withhold upstream body text and all other upstream headers; the client receives a fixed generic message and synthetic refusals stay distinct. SSE diagnostics and xAI tool-envelope filtering precede client delivery/continuation storage. | | `policy-refusal.ts` | Rewrites an allowlisted non-combo HTTP 403 model refusal (`isUpstreamPolicyRefusal` in `src/lib/errors.ts`) from an xAI destination only (`isXaiResponsesDestination`: api.x.ai or the Grok CLI proxy, on either wire) to an HTTP 200 Responses `incomplete` / `content_filter` payload, JSON or SSE, for both `adapter-dispatch.ts` and `passthrough-delivery.ts`. A streamed rewrite takes the turn admission lease and releases it when the body finishes, so the refusal stays inside active-turn accounting. Combo attempts keep the original 403 so failover classifies it as a hop. | -| `sidecar-execution.ts`, `sidecar-send-budget.ts` | Image/video versus web-search execution, shared rotation, and adapter-aware prepaid inference/producer ownership under the [spend contract](responses-spend.md#prepaid-initial-sends). | +| `sidecar-execution.ts`, `sidecar-send-budget.ts` | Image/video versus web-search execution, per-build adapter prompt estimates for usage logging, shared rotation, and adapter-aware prepaid inference/producer ownership under the [spend contract](responses-spend.md#prepaid-initial-sends). | | `completion-policy.ts`, `run-turn-execution.ts` | Empty-completion eligibility and adapter-owned event turns; refreshed retries reject previously failed key bindings and revalidate after each queued admission; adapter-owned CLI spawns receive the post-setup `IncomingMeta.beforePhysicalSend` fence, with [replacement-key readmission](responses-failover.md#upstream-key-attempt-accounting). | | `adapter-dispatch.ts` | Translated initial dispatch, bounded recovery (including [Anthropic thinking-signature recovery](anthropic-thinking-recovery.md)) and the shared continuation retry counter. Initial and rebuilt adapter budget refusals retain the HTTP 413 [byte-accounting contract](byte-accounting.md#request-copy-accounting); ordinary build errors remain 400. | | `adapter-continuation.ts`, `adapter-delivery.ts` | Continuation event sources and final streaming/buffered bridging; a streamed turn with a `clientEncoder` option is handed to `src/server/inference/client-encoder-delivery.ts` instead of the bridge. | diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 31970b8bde5..37ba90afe43 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1108,7 +1108,7 @@ "responses-compaction-policy-identity.test.ts": "responses", "responses-compaction-http-isolation.test.ts": "responses", "responses-compaction-routing.test.ts": "responses", "native-compaction-gateway.test.ts": "responses", "native-compaction-gateway-policy.test.ts": "responses", "native-compaction-provenance.test.ts": "responses", "native-compaction-oauth.test.ts": "responses", "native-compaction-gateway-credentials.test.ts": "responses", "responses-compaction.test.ts": "responses", "responses-console-go-upload-retry.test.ts": "responses", - "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", + "responses-context-overflow.test.ts": "responses", "responses-sidecar-context-overflow.test.ts": "responses", "responses-sidecar-usage.test.ts": "responses", "responses-continuation-boundaries.test.ts": "responses", "responses-core-modules.test.ts": "responses", "responses-custom-tool-guidance.test.ts": "responses", "responses-custom-tool-historical-replay.test.ts": "responses", "responses-custom-tool-repair-dispatch.test.ts": "responses", "responses-custom-tool-repair.test.ts": "responses", diff --git a/tests/responses/responses-sidecar-usage.test.ts b/tests/responses/responses-sidecar-usage.test.ts new file mode 100644 index 00000000000..157ac20a0e2 --- /dev/null +++ b/tests/responses/responses-sidecar-usage.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { KIRO_COMPLETION_TOOL_NAME } from "../../src/adapters/kiro-constants"; +import { encodeMessage } from "../../src/lib/eventstream-decoder"; +import { saveCredential } from "../../src/oauth/store"; +import { handleResponses } from "../../src/server/responses/core"; +import { responseWithDeferredRequestLog } from "../../src/server/relay"; +import type { RequestLogContext } from "../../src/server/request-log"; +import { readUsageEntries, resetUsageReadCacheForTests } from "../../src/usage/log"; +import { summarizeUsage } from "../../src/usage/summary"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const originalFetch = globalThis.fetch; +const originalHome = process.env.OPENCODEX_HOME; +let home: string; +let isolated: IsolatedCodexHome; +let release: (() => void) | undefined; + +beforeEach(async () => { + home = mkdtempSync(join(tmpdir(), "ocx-sidecar-usage-")); + isolated = installIsolatedCodexHome("ocx-sidecar-usage-codex-"); + process.env.OPENCODEX_HOME = home; + release = acquireOwnedSpendHome(); + resetUsageReadCacheForTests(); + await saveCredential("kiro", { + access: "synthetic-kiro-access", refresh: "synthetic-kiro-refresh", + expires: Date.now() + 3_600_000, source: "oauth", + kiro: { apiRegion: "us-east-1" }, + }); +}); + +afterEach(() => { + release?.(); + release = undefined; + globalThis.fetch = originalFetch; + resetUsageReadCacheForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + isolated.restore(); + removeTreeWithRetry(home); +}); + +/** Build the synthetic Kiro completion frames consumed by all three dispatch fixtures. */ +function completedAnswer(): Response { + const frame = (type: string, value: Record) => encodeMessage( + { ":message-type": "event", ":event-type": type }, + new TextEncoder().encode(JSON.stringify(value)), + ); + const tool = { name: KIRO_COMPLETION_TOOL_NAME, toolUseId: "synthetic-completion" }; + return new Response(Buffer.concat([ + frame("toolUseEvent", tool), + frame("toolUseEvent", { ...tool, input: JSON.stringify({ answer: "Completed fixture answer." }) }), + frame("toolUseEvent", { ...tool, stop: true }), + ]), { headers: { "content-type": "application/vnd.amazon.eventstream" } }); +} + +describe("sidecar prompt usage accounting", () => { + for (const sidecar of ["none", "web-search", "images"] as const) { + test(`${sidecar}: persists the full Kiro prompt estimate into daily activity`, async () => { + let sends = 0; + globalThis.fetch = (async input => { + const url = input instanceof Request ? input.url : String(input); + if (!/^https:\/\/runtime\.[a-z0-9-]+\.kiro\.dev\/$/.test(url)) { + throw new Error("Unexpected network target blocked by sidecar usage fixture"); + } + sends++; + return completedAnswer(); + }) as typeof fetch; + const config = { + defaultProvider: "kiro", emptyCompletionRetry: false, + webSearchSidecar: { backend: "exa", exaApiKey: "synthetic-search-key" }, + images: { bridgeEnabled: true }, + providers: { + kiro: { adapter: "kiro", baseUrl: "https://runtime.us-east-1.kiro.dev", + authMode: "oauth", models: ["claude-opus-5.5"] }, + xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", + authMode: "key", apiKey: "synthetic-image-key", models: ["fixture"] }, + }, + } as OcxConfig; + const request = new Request("http://127.0.0.1/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "kiro/claude-opus-5.5", stream: true, store: false, + input: [ + { role: "user", content: "Earlier fixture context. ".repeat(4096) }, + { role: "assistant", content: "Earlier fixture answer." }, + { role: "user", content: "Continue." }, + ], + tools: [ + { type: "function", name: "fixture_lookup", parameters: { type: "object", properties: {} } }, + ...(sidecar === "none" ? [] : [{ type: sidecar === "web-search" ? "web_search" : "image_generation" }]), + ], + }), + }); + const log: RequestLogContext = { model: "", provider: "" }; + const started = Date.now(); + const response = await handleResponses(request, config, log); + const logged = responseWithDeferredRequestLog(response, `sidecar-usage-${sidecar}`, started, log); + expect(response.status).toBe(200); + expect(await logged.text()).toContain("Completed fixture answer."); + expect(sends).toBe(1); + + const entries = readUsageEntries(); + expect(entries).toHaveLength(1); + expect(entries[0]!.usageStatus).toBe("estimated"); + // The newest user message is tiny; the persisted request and physical attempt must + // still include the replayed prompt, just as the ordinary dispatch path does. + expect(entries[0]!.usage!.inputTokens).toBeGreaterThan(20_000); + expect(entries[0]!.attempts![0]!.usage!.inputTokens).toBeGreaterThan(20_000); + const summary = summarizeUsage(entries, "today", Date.now()); + expect(summary.summary.requests).toBe(1); + expect(summary.summary.totalTokens).toBe(entries[0]!.totalTokens!); + expect(summary.days[0]!.totalTokens).toBeGreaterThan(20_000); + expect(summary.models[0]!.inputTokens).toBeGreaterThan(20_000); + }); + } +}); From 416075fa24d90285b0f3a9baec8654dc78b16c4c Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:23:43 +0900 Subject: [PATCH 2/2] test: register the sidecar usage test in an existing layout shard Avoid a new explicitShards entry that collides with #7635's layout-server shard. Co-authored-by: Martin Becerra <197874+martinbcg@users.noreply.github.com> --- scripts/test-layout/layout-adapters.json | 1 + scripts/test-layout/layout-usage.json | 3 --- scripts/test-layout/layout.json | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 scripts/test-layout/layout-usage.json diff --git a/scripts/test-layout/layout-adapters.json b/scripts/test-layout/layout-adapters.json index e1e5a5b00ba..08f0ab47160 100644 --- a/scripts/test-layout/layout-adapters.json +++ b/scripts/test-layout/layout-adapters.json @@ -2,6 +2,7 @@ "cursor-native-grep-globs.test.ts": "providers/cursor", "tool-identity-hash.test.ts": "adapters", "eventstream-fragmentation.test.ts": "responses", + "responses-sidecar-usage.test.ts": "responses", "google-vertex-key-project.test.ts": "adapters/google", "provider-quota-abliteration.test.ts": "providers", "command-code-plan-denial.test.ts": "providers" diff --git a/scripts/test-layout/layout-usage.json b/scripts/test-layout/layout-usage.json deleted file mode 100644 index 5e5f2d85fb0..00000000000 --- a/scripts/test-layout/layout-usage.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "responses-sidecar-usage.test.ts": "responses" -} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b8d73e0cc93..f3a4bcc5fd8 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-usage.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config",