From 8ffd532365bf9f8bc78b024ed3e9c5116675e42e Mon Sep 17 00:00:00 2001 From: cpa Date: Wed, 11 Nov 2026 03:17:04 +0900 Subject: [PATCH] fix(logs): keep the provider's closed error class and finishReason on failure rows A Vertex/Gemini rejection reached the request log as a bare request_rejected: Google puts its class in error.status (not error.type), the Vertex and Antigravity transports rewrite the body for display before dispatch reads it, and a mid-stream error frame or a truncated turn kept only English text on the context. Rows now carry the closed class (Anthropic envelope types plus the canonical google.rpc.Code names) and the Gemini FinishReason member on failureDetail, never the provider message, and a later successful replay retracts the detail. Refs #8446 --- .../src/content/docs/guides/web-dashboard.md | 2 +- gui/src/pages/logs-failure-summary.ts | 4 +- scripts/test-layout/layout-usage.json | 1 + src/adapters/google-truncation.ts | 6 +- src/adapters/google.ts | 7 +- src/adapters/upstream-http-error.ts | 6 +- src/lib/failure-detail.ts | 75 ++++++++- src/server/chat-native.ts | 4 +- src/server/request-log-failure-detail.ts | 60 ++++++- src/server/responses/adapter-dispatch.ts | 9 +- src/server/responses/core-combo-failure.ts | 7 +- src/server/responses/request-transport.ts | 3 +- structure/data-planes/error-detail.md | 22 ++- tests/fixtures/test-layout-expected.json | 2 +- tests/usage/provider-failure-detail.test.ts | 151 ++++++++++++++++++ 15 files changed, 336 insertions(+), 23 deletions(-) create mode 100644 tests/usage/provider-failure-detail.test.ts diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index c05591eb93e..2344a57ac40 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -192,7 +192,7 @@ usage in the report. ### Filtering request logs -Request details prefer a recognized upstream failure cause over generic proxy-error copy. Physical sends are always shown when valid; missing or invalid accounting shows **Unavailable**, not zero. **Retries** counts sends after the first only when one fully settled, consistent attempt is recorded. Multi-attempt, unresolved, inconsistent, or lossy-normalized records omit retries instead of inventing them. Unknown persisted causes use the status-based headline without raw attribution. **Upstream reason** shows a classified provider condition (for example rate limited, quota or credits exhausted, context window exceeded, location not supported, account verification required, or model unavailable), and **Failure detail** shows text written by the proxy, such as a stall timeout or byte-cap limit; new rows do not store the provider's own error text. Older `usage.jsonl` files may still contain it, but the dashboard drops it when reading them. +Request details prefer a recognized upstream failure cause over generic proxy-error copy. Physical sends are always shown when valid; missing or invalid accounting shows **Unavailable**, not zero. **Retries** counts sends after the first only when one fully settled, consistent attempt is recorded. Multi-attempt, unresolved, inconsistent, or lossy-normalized records omit retries instead of inventing them. Unknown persisted causes use the status-based headline without raw attribution. **Upstream reason** shows a classified provider condition (for example rate limited, quota or credits exhausted, context window exceeded, location not supported, account verification required, or model unavailable), and **Failure detail** shows text written by the proxy, such as a stall timeout or byte-cap limit. When the provider names a published error class or finish reason (for example Gemini or Vertex `INVALID_ARGUMENT`, or `MALFORMED_FUNCTION_CALL`), Failure detail also shows that token; new rows do not store the provider's own error text. Older `usage.jsonl` files may still contain it, but the dashboard drops it when reading them. A proxy replay refusal keeps HTTP 429 and error code `upstream_reset_replay_refused`, but its request and attempt failure cause is `transport-ambiguous`, with resend permission diff --git a/gui/src/pages/logs-failure-summary.ts b/gui/src/pages/logs-failure-summary.ts index 0dbc43c29dc..b263371396b 100644 --- a/gui/src/pages/logs-failure-summary.ts +++ b/gui/src/pages/logs-failure-summary.ts @@ -157,7 +157,9 @@ export function failureDetailLabel(value: unknown, t: TFn): string | undefined { const detail = boundedFailureDetail(value); if (!detail) return undefined; if (detail.code === "provider_error") { - const label = detail.type ? t("logs.failureDetail.providerErrorTyped", { type: detail.type }) : t(FAILURE_DETAIL_KEYS.provider_error); + const typed = detail.type ? t("logs.failureDetail.providerErrorTyped", { type: detail.type }) : t(FAILURE_DETAIL_KEYS.provider_error); + // The provider's closed finishReason is a machine token, appended like the validation pairs. + const label = detail.finishReason ? `${typed} [finishReason: ${detail.finishReason}]` : typed; return detail.validation ? `${label}: ${detail.validation.map(issue => `${issue.field} [${issue.type}]`).join("; ")}` : label; } if (detail.code === "header_timeout" && detail.value !== undefined) { diff --git a/scripts/test-layout/layout-usage.json b/scripts/test-layout/layout-usage.json index 26ebfe57b7e..508b645fd67 100644 --- a/scripts/test-layout/layout-usage.json +++ b/scripts/test-layout/layout-usage.json @@ -4,6 +4,7 @@ "request-log.test.ts": "usage", "request-log-normalization.test.ts": "usage", "request-log-error-detail.test.ts": "usage", + "provider-failure-detail.test.ts": "usage", "usage-telemetry-profile-contract.test.ts": "usage", "usage-telemetry-profile-sync.test.ts": "usage", "usage-telemetry-profile-collector.test.ts": "usage", diff --git a/src/adapters/google-truncation.ts b/src/adapters/google-truncation.ts index 0e8739e5e1c..a8629fd14a2 100644 --- a/src/adapters/google-truncation.ts +++ b/src/adapters/google-truncation.ts @@ -1,4 +1,5 @@ import { redactSecretString } from "../lib/redact"; +import { withProviderFailureFacts } from "../lib/failure-detail"; import type { AdapterEvent, OcxUsage } from "../types"; /** Gemini/Vertex finishReason values that mean the turn was cut off, not cleanly stopped. */ @@ -37,10 +38,11 @@ export function vertexTruncationErrorEvent( toolCallsStarted: number, usage?: OcxUsage, ): Extract { - return { + // The request log keeps the closed finish reason that ended the turn (#8446); the wire event is unchanged. + return withProviderFailureFacts({ type: "error", message: vertexTruncationErrorMessage(finishReason), ...(usage ? { usage } : {}), ...(finishReason === "MALFORMED_FUNCTION_CALL" && toolCallsStarted === 0 ? { replaySafeBeforeOutput: true } : {}), - }; + } as const, { finishReason }); } diff --git a/src/adapters/google.ts b/src/adapters/google.ts index a449bc97946..668b66a2d7e 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -3,6 +3,7 @@ import type { AdapterFetchContext, AdapterRequest, IncomingMeta, ProviderAdapter import { debugDroppedFrame, debugProviderDiagnosticLazy } from "../lib/debug"; import { createToolCallIdAllocator } from "./tool-call-id"; import { openCodeGoGeminiEndpoint } from "../types/opencode-go"; +import { withProviderFailureFacts } from "../lib/failure-detail"; import { createImageBudget, materializeInlineImage, MAX_ENCODED_BYTES_PER_IMAGE, artifactHttpUrl } from "../images/artifacts"; import type { AdapterEvent, @@ -1393,12 +1394,14 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte // Inline provider error inside a 200 stream → terminal error (see openai-chat.ts). if (chunk.error) { - const err = chunk.error as { message?: string } | undefined; + const err = chunk.error as { message?: string; status?: unknown } | undefined; // Clear-on-invalid: a signature rejection means our replayed thoughtSignatures are stale. // Drop the cache entry and durable store entry for rejected calls so the next turn // starts clean instead of re-injecting a bad sig. handleSignatureRejection(err?.message); - yield { type: "error", message: err?.message ?? "upstream error" }; + // Only the closed RPC status class and finish reason ride along for the request log (#8446). + yield withProviderFailureFacts({ type: "error", message: err?.message ?? "upstream error" } as AdapterEvent, + { type: err?.status, finishReason: lastFinishReason }); return "terminate"; } diff --git a/src/adapters/upstream-http-error.ts b/src/adapters/upstream-http-error.ts index 25b49e38c4b..733ecbbc06a 100644 --- a/src/adapters/upstream-http-error.ts +++ b/src/adapters/upstream-http-error.ts @@ -1,5 +1,6 @@ import { inheritAccountAutoPauseResponse } from "../lib/account-auto-pause-response"; import { readBoundedResponseBody } from "../lib/bounded-body"; +import { providerErrorTypeOf, withProviderFailureFacts } from "../lib/failure-detail"; import { redactSecretString } from "../lib/redact"; const ABSOLUTE_PATH_PATTERN = /(?:\/Users\/[^ "';,]+|\/home\/[^ "';,]+|\/root\/[^ "';,]*|[A-Za-z]:\\Users\\[^ "';,]+)/g; @@ -41,9 +42,10 @@ export async function normalizeUpstreamHttpErrorResponse( const headers = new Headers(res.headers); headers.delete("content-encoding"); headers.delete("content-length"); - return inheritAccountAutoPauseResponse(res, new Response(await opts.formatMessage(payloadText), { + // The rewritten body is display text; keep the provider's closed error class for the request log (#8446). + return withProviderFailureFacts(inheritAccountAutoPauseResponse(res, new Response(await opts.formatMessage(payloadText), { status: res.status, statusText: res.statusText, headers, - })); + })), { type: providerErrorTypeOf(parseUpstreamJsonPayload(payloadText)) }); } diff --git a/src/lib/failure-detail.ts b/src/lib/failure-detail.ts index 6fc01807904..f7ed1ef4703 100644 --- a/src/lib/failure-detail.ts +++ b/src/lib/failure-detail.ts @@ -23,24 +23,43 @@ export const FAILURE_DETAIL_CODES = [ export type FailureDetailCode = typeof FAILURE_DETAIL_CODES[number]; -/** Closed provider error classes a provider_error detail may name (the Anthropic error envelope types). */ +/** + * Closed provider error classes a provider_error detail may name: the Anthropic error envelope types + * and the canonical Google RPC status names (google.rpc.Code) that Gemini, Vertex and Antigravity put in + * `error.status`. Both are published vocabularies, so naming one discloses nothing the provider wrote. + */ export const FAILURE_DETAIL_PROVIDER_TYPES = [ "invalid_request_error", "authentication_error", "permission_error", "not_found_error", "rate_limit_error", "api_error", "overloaded_error", "request_too_large", + "CANCELLED", "UNKNOWN", "INVALID_ARGUMENT", "DEADLINE_EXCEEDED", "NOT_FOUND", "ALREADY_EXISTS", + "PERMISSION_DENIED", "RESOURCE_EXHAUSTED", "FAILED_PRECONDITION", "ABORTED", "OUT_OF_RANGE", + "UNIMPLEMENTED", "INTERNAL", "UNAVAILABLE", "DATA_LOSS", "UNAUTHENTICATED", ] as const; export type FailureDetailProviderType = typeof FAILURE_DETAIL_PROVIDER_TYPES[number]; +/** Gemini/Vertex candidate finishReason values (the published FinishReason enum) a provider_error may name. */ +export const FAILURE_DETAIL_FINISH_REASONS = [ + "FINISH_REASON_UNSPECIFIED", "STOP", "MAX_TOKENS", "SAFETY", "RECITATION", "LANGUAGE", "OTHER", "BLOCKLIST", + "PROHIBITED_CONTENT", "SPII", "MALFORMED_FUNCTION_CALL", "IMAGE_SAFETY", "IMAGE_PROHIBITED_CONTENT", + "IMAGE_RECITATION", "IMAGE_OTHER", "NO_IMAGE", "UNEXPECTED_TOOL_CALL", "TOO_MANY_TOOL_CALLS", +] as const; + +export type FailureDetailFinishReason = typeof FAILURE_DETAIL_FINISH_REASONS[number]; + export interface FailureDetail { code: FailureDetailCode; /** Milliseconds for timeouts and stalls, bytes for an overflow. */ value?: number; type?: FailureDetailProviderType; + /** The provider's own closed finish reason for the turn that failed; provider_error only. */ + finishReason?: FailureDetailFinishReason; validation?: ValidationIssue[]; } const CODES: ReadonlySet = new Set(FAILURE_DETAIL_CODES); const TYPES: ReadonlySet = new Set(FAILURE_DETAIL_PROVIDER_TYPES); +const FINISH_REASONS: ReadonlySet = new Set(FAILURE_DETAIL_FINISH_REASONS); const MAX_VALUE = 1e12; /** Codes whose value parameter carries meaning; the value is dropped for any other code. */ @@ -61,9 +80,59 @@ export function boundedFailureDetail(value: unknown): FailureDetail | undefined const valid = typeof number === "number" && Number.isSafeInteger(number) && number >= 0 && number <= MAX_VALUE && VALUED.has(code); const type = code === "provider_error" && typeof record.type === "string" && TYPES.has(record.type) ? record.type as FailureDetailProviderType : undefined; + const finishReason = code === "provider_error" && typeof record.finishReason === "string" && FINISH_REASONS.has(record.finishReason) + ? record.finishReason as FailureDetailFinishReason : undefined; const validation = code === "provider_error" ? boundedValidationIssues(record.validation) : undefined; const keys = Object.keys(record); - const exact = keys.every(key => key === "code" || (key === "value" && valid) || (key === "type" && type !== undefined) || (key === "validation" && validation !== undefined && validation === record.validation)); + const exact = keys.every(key => key === "code" || (key === "value" && valid) || (key === "type" && type !== undefined) + || (key === "finishReason" && finishReason !== undefined) + || (key === "validation" && validation !== undefined && validation === record.validation)); if (exact) return record as unknown as FailureDetail; - return { code, ...(valid ? { value: number as number } : {}), ...(type ? { type } : {}), ...(validation ? { validation } : {}) }; + return { + code, ...(valid ? { value: number as number } : {}), ...(type ? { type } : {}), + ...(finishReason ? { finishReason } : {}), ...(validation ? { validation } : {}), + }; +} + +/** The closed facts an adapter can state about a provider failure it parsed off the wire. */ +export interface ProviderFailureFacts { + type?: FailureDetailProviderType; + finishReason?: FailureDetailFinishReason; +} + +// Keyed by the adapter event object, so the facts can never be serialized with it; an event a guard +// rebuilds simply carries none. +const providerFailureFacts = new WeakMap(); + +/** + * Attach the closed provider facts to an adapter error event. Values outside the closed sets are + * dropped here, so an adapter can pass what the provider sent without checking it first. + */ +export function withProviderFailureFacts(event: T, facts: { type?: unknown; finishReason?: unknown }): T { + const type = typeof facts.type === "string" && TYPES.has(facts.type) ? facts.type as FailureDetailProviderType : undefined; + const finishReason = typeof facts.finishReason === "string" && FINISH_REASONS.has(facts.finishReason) + ? facts.finishReason as FailureDetailFinishReason : undefined; + if (type || finishReason) providerFailureFacts.set(event, { ...(type ? { type } : {}), ...(finishReason ? { finishReason } : {}) }); + return event; +} + +export function providerFailureDetailOf(event: object): ProviderFailureFacts | undefined { + return providerFailureFacts.get(event); +} + +const isObject = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +/** + * The closed provider error class a parsed upstream error body names: the envelope `error.type`, Google's + * `error.status` (also as the first element of a streamed JSON array), or a top-level `type`. + */ +export function providerErrorTypeOf(payload: unknown): FailureDetailProviderType | undefined { + const record = Array.isArray(payload) ? payload[0] : payload; + if (!isObject(record)) return undefined; + const nested = isObject(record.error) ? record.error : undefined; + for (const value of [nested?.type, nested?.status, record.type]) { + if (typeof value === "string" && TYPES.has(value)) return value as FailureDetailProviderType; + } + return undefined; } diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index c25f4909ae2..1c038f394bf 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -1,4 +1,4 @@ -import { recordUpstreamValidationDetail } from "./request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail } from "./request-log-failure-detail"; import { normalizeNativeChatContent } from "../adapters/openai-chat/native-content"; import { attachComboFailureEvidence, ownedChatErrorPayload, type ClientErrorContract } from "../lib/client-error-body"; import type { FailureVerdictSink } from "../chat/failure-verdict"; @@ -754,7 +754,7 @@ export async function runNativeChatAttempt( let upstreamMessage: string | undefined; try { const parsedError = JSON.parse(bodyText) as Rec; - recordUpstreamValidationDetail(logCtx, response.status, parsedError); + recordUpstreamProviderErrorDetail(logCtx, response.status, parsedError); const nested = isRec(parsedError.error) ? parsedError.error : undefined; const details = nested ?? parsedError; if (typeof details.type === "string") upstreamType = details.type; diff --git a/src/server/request-log-failure-detail.ts b/src/server/request-log-failure-detail.ts index e521be7b69f..05b6829bac7 100644 --- a/src/server/request-log-failure-detail.ts +++ b/src/server/request-log-failure-detail.ts @@ -10,7 +10,7 @@ import { knownModelIdentity } from "../lib/model-identity"; * upstreamError, which internal classifiers and status heuristics read and which never reaches a row. * Only proxy-composed text may be passed here. */ -import { boundedFailureDetail, type FailureDetail, type FailureDetailCode } from "../lib/failure-detail"; +import { boundedFailureDetail, providerErrorTypeOf, providerFailureDetailOf, type FailureDetail, type FailureDetailCode } from "../lib/failure-detail"; export type { FailureDetail }; @@ -95,3 +95,61 @@ export function recordUpstreamValidationDetail(logCtx: { failureDetail?: Failure ? record.error as Record : undefined; logCtx.failureDetail = boundedFailureDetail({ code: "provider_error", type: nested?.type ?? record.type, validation }); } + +/** + * Capture the closed provider error class from a raw upstream error body (#8446): the envelope + * `type` (Anthropic/OpenAI) or Google's `error.status`, the latter also as the first element of a + * streamed JSON array. Validation pairs keep their existing capture and win when present. Only + * callers holding the provider's own body may use this; a body this proxy formatted is not evidence. + * An adapter transport that rewrote the body (normalizeUpstreamHttpErrorResponse) attached the class + * it read from the original to `response`, which is preferred. + */ +export function recordUpstreamProviderErrorDetail( + logCtx: { failureDetail?: FailureDetail }, + status: number, + payload: unknown, + response?: object, +): void { + if (status < 400 || status >= 600) return; + const before = logCtx.failureDetail; + recordUpstreamValidationDetail(logCtx, status, payload); + if (logCtx.failureDetail !== before) { providerDetails.set(logCtx, logCtx.failureDetail!); return; } + let type = response ? providerFailureDetailOf(response)?.type : undefined; + if (type === undefined) { + let parsed = payload; + if (typeof payload === "string") { + if (payload.length > 256 * 1024) return; + try { parsed = JSON.parse(payload); } catch { return; } + } + type = providerErrorTypeOf(parsed); + } + if (type === undefined) return; + const detail: FailureDetail = { code: "provider_error", type }; + logCtx.failureDetail = detail; + providerDetails.set(logCtx, detail); +} + +/** The provider detail each context last recorded, so a later success can retract exactly that one. */ +const providerDetails = new WeakMap(); + +/** + * Request-log side of every adapter event (the parse seam in request-transport.ts). An error event an + * adapter tagged with closed provider facts records them as a provider_error detail; a later `done` on + * the same request (a retry or replay that succeeded) retracts a detail this path recorded, so a + * successful row never names a provider failure it recovered from. + */ +export function observeProviderFailureEvent(logCtx: { failureDetail?: FailureDetail }, event: { type: string }): void { + if (event.type === "done") { + const recorded = providerDetails.get(logCtx); + if (recorded !== undefined && logCtx.failureDetail === recorded) delete logCtx.failureDetail; + providerDetails.delete(logCtx); + return; + } + if (event.type !== "error") return; + const facts = providerFailureDetailOf(event); + if (!facts) return; + const detail = boundedFailureDetail({ code: "provider_error", ...facts }); + if (!detail) return; + logCtx.failureDetail = detail; + providerDetails.set(logCtx, detail); +} diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 2088d3ec6ae..5954eedd158 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -1,8 +1,8 @@ -import { recordUpstreamValidationDetail } from "../request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail } from "../request-log-failure-detail"; import { clientFacingModelText } from "../../lib/error-destination"; import { responsesSpendTarget } from "./request-spend-target"; import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; -import { markProxyError, carryProxyError } from "../../lib/client-error-body"; +import { markProxyError, carryProxyError, isProxyError } from "../../lib/client-error-body"; import { codexAdapterBuildRefusal } from "./passthrough-build-error"; import { createAutoPauseRefreshErrorMapper, recoverAutoPausedAccount } from "./account-auto-pause-recovery"; import { wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; @@ -1864,7 +1864,7 @@ export async function prepareAdapterExchange( // `response.body` itself with the abort signal threaded through, and the combo // contract is that this body's getter is touched exactly once. A guard here would be // a second `.body` access for no gain, since the bounded reader owns settlement. - const failure = await consumeComboFailure(upstreamResponse, options.abortSignal, Date.now(), logCtx) + const failure = await consumeComboFailure(upstreamResponse, options.abortSignal, Date.now(), logCtx, !isProxyError(upstreamResponse)) .finally(cleanupUpstreamAbort); options.onConsumedComboFailure?.(failure); return failure.response; @@ -1879,7 +1879,8 @@ export async function prepareAdapterExchange( } finally { cleanupUpstreamAbort(); } - recordUpstreamValidationDetail(logCtx, upstreamResponse.status, errorText); + // The provider's own body (not a proxy-normalized one) may name its closed error class (#8446). + if (!isProxyError(upstreamResponse)) recordUpstreamProviderErrorDetail(logCtx, upstreamResponse.status, errorText, upstreamResponse); if (upstreamResponse.status === 413) { if (options.clientModelLabel) projectLocalModelRefusalLog(logCtx, options.clientModelLabel); return clientRequestedStream diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index d3bbe298afe..686bd3583a7 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -1,4 +1,4 @@ -import { recordUpstreamValidationDetail, type FailureDetail } from "../request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail, recordUpstreamValidationDetail, type FailureDetail } from "../request-log-failure-detail"; import { carryProxyErrorPresentation } from "../../lib/client-error-body"; import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { comboFailureEvidence, carryProxyError } from "../../lib/client-error-body"; @@ -47,6 +47,8 @@ export async function consumeComboFailure( signal?: AbortSignal, now = Date.now(), logCtx?: { failureDetail?: FailureDetail; causeHint?: RequestFailureCause }, + /** True only when `response` is the provider's own HTTP answer, so its closed error class is evidence. */ + rawUpstream = false, ): Promise { // Read before the body: the marker lives on this Response object, and the failure below is // rebuilt as a new one that would otherwise lose it. @@ -85,7 +87,8 @@ export async function consumeComboFailure( && isRateLimitOrQuotaFailureMessage(quotaMessage); } if (body.displaySafe && !body.truncated) { - if (logCtx) recordUpstreamValidationDetail(logCtx, response.status, body.text); + if (logCtx && rawUpstream) recordUpstreamProviderErrorDetail(logCtx, response.status, body.text, response); + else if (logCtx) recordUpstreamValidationDetail(logCtx, response.status, body.text); // A combo child's row is attributed from the same evidence the parent uses: Meta's Muse // Code refusal is a plan restriction, not a rejected credential (#8352). if (logCtx) stampMetaMusePlanRefusal(logCtx, response.status, body.text); diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index c98ba4f0311..39b0d8951e0 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -1,5 +1,5 @@ import { publicOAuthErrorPresentation } from "../../oauth"; -import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; +import { observeProviderFailureEvent, projectLocalModelRefusalLog } from "../request-log-failure-detail"; import { markProxyError } from "../../lib/client-error-body"; import { formatClientRetryAfter } from "../../lib/client-retry-after"; import { SendBudgetExhaustedError, errorDestinationText, providerErrorDestinationLabel } from "../../lib/upstream-retry"; @@ -676,6 +676,7 @@ export async function prepareResponsesTransport( const observeEvent = (event: AdapterEvent, response: object): void => { delivery.noteAdapterEvent(); observeUsage(event, response); + observeProviderFailureEvent(logCtx, event); }; const parseStream = resolved.parseStream.bind(resolved); resolved.parseStream = async function* (...args) { diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index 561b51549ce..10724e0ec22 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -40,7 +40,9 @@ through the disk reader. A row keeps: evidence before discarding its text; `tests/usage/request-log-normalization.test.ts` covers both. - `failureDetail`, a closed `{ code, value?, type?, validation? }` from `src/lib/failure-detail.ts`: `code` is a member of `FAILURE_DETAIL_CODES`, `value` a safe integer (milliseconds or bytes) kept only for codes that take one, - and `type` a closed Anthropic error class kept only on `provider_error`. `recordLocalFailureDetail` in + `type` a closed provider error class (the Anthropic envelope types or a canonical Google RPC status + name such as `INVALID_ARGUMENT`) and `finishReason` a member of the Gemini/Vertex FinishReason enum, + both kept only on `provider_error`. `recordLocalFailureDetail` in `src/server/request-log-failure-detail.ts` takes the detail beside the English text it leaves on the context; `incompleteReasonDetail` maps the known `incomplete_details.reason` values (`max_output_tokens`, `upstream_stall_timeout`, `adapter_eof`), and an unknown reason stays on the context only. A relayed error @@ -62,6 +64,24 @@ partial bodies produce status-only diagnostics. GUI rendering. The dashboard appends the machine field/type pairs to its localized provider-error label. No upstream-authored free text is retained, and client-facing messages retain their existing policy. +A provider failure also keeps its closed class without validation pairs (#8446). +`recordUpstreamProviderErrorDetail` reads `error.type`, Google's `error.status` (an object or the first +element of a streamed array) or a top-level `type` from a raw upstream body: native Chat, the +translated adapter dispatch, and `consumeComboFailure` when adapter dispatch passes it the provider's +own response. A proxy-marked response and every other `consumeComboFailure` caller keep the +validation-only capture, because a body this proxy formatted is not provider evidence. An adapter +transport that rewrites the body for display (`normalizeUpstreamHttpErrorResponse`, used by Vertex and +Antigravity) reads the class from the original first and attaches it to the new response through +`withProviderFailureFacts` in `src/lib/failure-detail.ts`, a `WeakMap` keyed by the object, so +nothing new is serialized. The Google adapter tags a mid-stream `error` frame with its status and a +fail-closed Vertex truncation with its finish reason the same way, and the parse seam in +`src/server/responses/request-transport.ts` calls `observeProviderFailureEvent`, which records them. +A later `done` on the same request (a retry or empty-completion replay that succeeded) retracts the +detail this path recorded, so a successful row never names a failure it recovered from. The message +itself stays on the context like every other upstream text. The dashboard appends the finish reason as +a machine token. `tests/usage/provider-failure-detail.test.ts` drives the Vertex route for an HTTP 400, +a combo child, a mid-stream error frame, a truncated turn and a recovered replay. + `upstreamError`, `upstreamErrorCode`, `upstreamErrorType` and `upstreamRequestId` stay on the request context, where status heuristics and classifiers read them, and never reach a row, `usage.jsonl` or the service log. Bytes already in `usage.jsonl` from earlier builds are not rewritten; they are dropped on every diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 25ba4fd2811..6744c006765 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1624,7 +1624,7 @@ "hermes-profiles.test.ts": "clients", "hermes-profile-routes.test.ts": "clients" ,"cli-sync-expect-link.test.ts": "cli" ,"local-credentials.test.ts": "cli", "local-credential-generation.test.ts": "config", "link-connection-generation.test.ts": "clients" - ,"request-log-error-detail.test.ts": "usage", "upstream-condition.test.ts": "lib", "client-retry-after.test.ts": "lib", "client-retry-after-sites.test.ts": "server", "upstream-error-detail-boundaries.test.ts": "server", "request-log-identity-bounds.test.ts": "usage", "safe-response-headers.test.ts": "server", "turn-state-handles.test.ts": "server", "turn-state-handles-e2e.test.ts": "server", "client-error-body.test.ts": "lib", "chat-error-body-normalization.test.ts": "responses", "chat-error-collector-verdicts.test.ts": "responses", "server-chat-error-body-evidence.test.ts": "server" + ,"request-log-error-detail.test.ts": "usage", "provider-failure-detail.test.ts": "usage", "upstream-condition.test.ts": "lib", "client-retry-after.test.ts": "lib", "client-retry-after-sites.test.ts": "server", "upstream-error-detail-boundaries.test.ts": "server", "request-log-identity-bounds.test.ts": "usage", "safe-response-headers.test.ts": "server", "turn-state-handles.test.ts": "server", "turn-state-handles-e2e.test.ts": "server", "client-error-body.test.ts": "lib", "chat-error-body-normalization.test.ts": "responses", "chat-error-collector-verdicts.test.ts": "responses", "server-chat-error-body-evidence.test.ts": "server" ,"messages-error-normalization.test.ts": "claude-integration", "messages-header-normalization.test.ts": "claude-integration", "messages-lane-errors.test.ts": "claude-integration" ,"messages-error-contracts.test.ts": "claude-integration", "messages-overflow-verdicts.test.ts": "claude-integration", "messages-review-regressions.test.ts": "claude-integration", "messages-proxy-refusals.test.ts": "claude-integration", "messages-overflow-native.test.ts": "claude-integration", "messages-overflow-builtin.test.ts": "claude-integration", "messages-overflow-translated.test.ts": "claude-integration", "messages-overflow-stream.test.ts": "claude-integration", "messages-overflow-collectors.test.ts": "claude-integration", "messages-success-media-types.test.ts": "claude-integration", "messages-native-stream-parity.test.ts": "claude-integration", "messages-overflow-baseline-contract.test.ts": "claude-integration", "messages-sse-builtin-errors.test.ts": "claude-integration", diff --git a/tests/usage/provider-failure-detail.test.ts b/tests/usage/provider-failure-detail.test.ts new file mode 100644 index 00000000000..fa5c03dcc35 --- /dev/null +++ b/tests/usage/provider-failure-detail.test.ts @@ -0,0 +1,151 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { handleResponses } from "../../src/server/responses"; +import { addFinalRequestLog, type RequestLogContext, type RequestLogEntry } from "../../src/server/request-log"; +import { recordUpstreamProviderErrorDetail } from "../../src/server/request-log-failure-detail"; +import { boundedFailureDetail, providerFailureDetailOf } from "../../src/lib/failure-detail"; +import { createGoogleAdapter } from "../../src/adapters/google"; +import { failureDetailLabel } from "../../gui/src/pages/logs-failure-summary"; +import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types"; + +const releaseSpendHome = acquireOwnedSpendHome(); +afterAll(() => releaseSpendHome()); + +// Upstream-authored text that must never reach a row: a credential-shaped token and prompt words. +const SECRET = "zq7Vb2LmN4pX9tR1sK8wD3fH6jY0cA5e"; +const MESSAGE = `Invalid value at contents[3] near "private prompt" for key ${SECRET}`; +const googleError = (status: string, code = 400) => ({ error: { code, status, message: MESSAGE } }); + +describe("closed provider failure detail contract (#8446)", () => { + test("keeps a Google RPC status class and a closed finishReason on provider_error", () => { + expect(boundedFailureDetail({ code: "provider_error", type: "INVALID_ARGUMENT" })).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(boundedFailureDetail({ code: "provider_error", type: "FAILED_PRECONDITION", finishReason: "MALFORMED_FUNCTION_CALL" })) + .toEqual({ code: "provider_error", type: "FAILED_PRECONDITION", finishReason: "MALFORMED_FUNCTION_CALL" }); + }); + + test("drops unknown statuses and reasons, free text, and a finishReason on another code", () => { + expect(boundedFailureDetail({ code: "provider_error", type: SECRET, finishReason: MESSAGE })).toEqual({ code: "provider_error" }); + expect(boundedFailureDetail({ code: "provider_error", type: "invalid_argument" })).toEqual({ code: "provider_error" }); + expect(boundedFailureDetail({ code: "stream_failed", finishReason: "SAFETY" })).toEqual({ code: "stream_failed" }); + }); + + test("the dashboard renders the status and the finish reason as machine tokens", () => { + const t = (key: string, vars?: Record) => `${key}${vars ? JSON.stringify(vars) : ""}`; + const label = failureDetailLabel({ code: "provider_error", type: "INVALID_ARGUMENT", finishReason: "MALFORMED_FUNCTION_CALL" }, t as never); + expect(label).toContain("INVALID_ARGUMENT"); + expect(label).toContain("MALFORMED_FUNCTION_CALL"); + }); +}); + +describe("raw upstream error capture", () => { + test("records the Google status class from an object or a streamed array body, never the message", () => { + for (const body of [googleError("INVALID_ARGUMENT"), [googleError("INVALID_ARGUMENT")]]) { + const ctx: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(ctx, 400, JSON.stringify(body)); + expect(ctx.failureDetail).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(JSON.stringify(ctx.failureDetail)).not.toContain(SECRET); + } + }); + + test("keeps existing closed envelope types and validation pairs, and ignores success or unknown classes", () => { + const typed: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(typed, 400, { error: { type: "invalid_request_error", message: MESSAGE } }); + expect(typed.failureDetail).toEqual({ code: "provider_error", type: "invalid_request_error" }); + const validation: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(validation, 422, { detail: [{ loc: ["body", "store"], type: "extra_forbidden", input: SECRET }] }); + expect(validation.failureDetail).toEqual({ code: "provider_error", validation: [{ field: "store", type: "extra_forbidden" }] }); + for (const [status, body] of [[200, googleError("INVALID_ARGUMENT")], [400, { error: { type: SECRET, message: MESSAGE } }], [400, MESSAGE]] as const) { + const ctx: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(ctx, status, typeof body === "string" ? body : JSON.stringify(body)); + expect(ctx.failureDetail).toBeUndefined(); + } + const server: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(server, 503, JSON.stringify(googleError("UNAVAILABLE", 503))); + expect(server.failureDetail).toEqual({ code: "provider_error", type: "UNAVAILABLE" }); + }); +}); + +describe("Google adapter failure events", () => { + const vertex = { adapter: "google", baseUrl: "https://x", googleMode: "vertex" } as OcxProviderConfig; + const collect = async (chunks: unknown[]): Promise => { + const adapter = withTestTranslatorBudget(createGoogleAdapter(vertex)); + const body = chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join(""); + const events: AdapterEvent[] = []; + for await (const event of adapter.parseStream(new Response(body, { headers: { "content-type": "text/event-stream" } }))) events.push(event); + return events; + }; + + test("a mid-stream error frame carries its closed status class off the wire", async () => { + const events = await collect([googleError("FAILED_PRECONDITION")]); + const error = events.at(-1)!; + expect(error.type).toBe("error"); + expect(providerFailureDetailOf(error)).toEqual({ type: "FAILED_PRECONDITION" }); + expect(JSON.stringify(error)).not.toContain("FAILED_PRECONDITION\"}"); + }); + + test("a truncated turn carries the finishReason that ended it", async () => { + const events = await collect([{ candidates: [{ finishReason: "MALFORMED_FUNCTION_CALL" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 0 } }]); + expect(providerFailureDetailOf(events.at(-1)!)).toEqual({ finishReason: "MALFORMED_FUNCTION_CALL" }); + }); +}); + +describe("Responses rows through the Vertex route", () => { + // The Vertex transport builds its own googleapis.com URL, so the fixture answers at fetch itself: + // nothing in these cases may leave the process. + async function run(answer: () => Response, options: { config?: Partial; model?: string } = {}) { + const realFetch = globalThis.fetch; + let calls = 0; + globalThis.fetch = (async () => { calls += 1; return answer(); }) as unknown as typeof fetch; + try { + const config = { port: 0, defaultProvider: "fixture", ...options.config, providers: { fixture: { + adapter: "google", googleMode: "vertex", project: "fixture-project", location: "us-central1", + baseUrl: "https://aiplatform.googleapis.com", authMode: "key", apiKey: SECRET, + } } } as OcxConfig; + const log: RequestLogContext = { model: "gemini-3-pro", provider: "fixture" }; + const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: options.model ?? "fixture/gemini-3-pro", input: "private prompt", stream: true }) }), config, log); + await response.text().catch(() => ""); + let row: RequestLogEntry | undefined; + addFinalRequestLog("fixture", Date.now(), log, response.status === 200 ? 502 : response.status, undefined, entry => { row = entry; }); + return { log, row, calls }; + } finally { globalThis.fetch = realFetch; } + } + const sse = (...frames: unknown[]) => () => new Response(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(""), + { headers: { "content-type": "text/event-stream" } }); + const malformed = sse({ candidates: [{ finishReason: "MALFORMED_FUNCTION_CALL" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 0 } }); + + test("an HTTP 400 keeps INVALID_ARGUMENT in the row without the provider message", async () => { + const { row } = await run(() => Response.json(googleError("INVALID_ARGUMENT"), { status: 400 })); + expect(row?.failureDetail).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(JSON.stringify(row)).not.toContain(SECRET); + expect(JSON.stringify(row)).not.toContain("private prompt"); + }); + + test("a combo child keeps the class of the provider refusal", async () => { + const { log } = await run(() => Response.json(googleError("FAILED_PRECONDITION"), { status: 400 }), { + model: "pair", config: { combos: { pair: { strategy: "fallback", targets: [{ provider: "fixture", model: "gemini-3-pro" }] } } } as Partial, + }); + expect(log.failureDetail).toEqual({ code: "provider_error", type: "FAILED_PRECONDITION" }); + }); + + test("a mid-stream error frame keeps its status class in the row", async () => { + const { row } = await run(sse(googleError("FAILED_PRECONDITION"))); + expect(row?.failureDetail).toEqual({ code: "provider_error", type: "FAILED_PRECONDITION" }); + expect(JSON.stringify(row)).not.toContain(SECRET); + }); + + test("a turn the provider truncated keeps its finishReason in the row", async () => { + const { row } = await run(malformed); + expect(row?.failureDetail).toEqual({ code: "provider_error", finishReason: "MALFORMED_FUNCTION_CALL" }); + }); + + test("a replay that then succeeds leaves no provider failure detail behind", async () => { + let served = 0; + const ok = sse({ candidates: [{ content: { role: "model", parts: [{ text: "ok" }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 } }); + const { log, calls } = await run(() => (++served === 1 ? malformed() : ok()), { config: { emptyCompletionRetry: true } }); + expect(calls).toBe(2); + expect(log.failureDetail).toBeUndefined(); + }); +});