diff --git a/docs-site/src/content/docs/guides/web-dashboard.md b/docs-site/src/content/docs/guides/web-dashboard.md index c05591eb93e..2344a57ac40 100644 --- a/docs-site/src/content/docs/guides/web-dashboard.md +++ b/docs-site/src/content/docs/guides/web-dashboard.md @@ -192,7 +192,7 @@ usage in the report. ### Filtering request logs -Request details prefer a recognized upstream failure cause over generic proxy-error copy. Physical sends are always shown when valid; missing or invalid accounting shows **Unavailable**, not zero. **Retries** counts sends after the first only when one fully settled, consistent attempt is recorded. Multi-attempt, unresolved, inconsistent, or lossy-normalized records omit retries instead of inventing them. Unknown persisted causes use the status-based headline without raw attribution. **Upstream reason** shows a classified provider condition (for example rate limited, quota or credits exhausted, context window exceeded, location not supported, account verification required, or model unavailable), and **Failure detail** shows text written by the proxy, such as a stall timeout or byte-cap limit; new rows do not store the provider's own error text. Older `usage.jsonl` files may still contain it, but the dashboard drops it when reading them. +Request details prefer a recognized upstream failure cause over generic proxy-error copy. Physical sends are always shown when valid; missing or invalid accounting shows **Unavailable**, not zero. **Retries** counts sends after the first only when one fully settled, consistent attempt is recorded. Multi-attempt, unresolved, inconsistent, or lossy-normalized records omit retries instead of inventing them. Unknown persisted causes use the status-based headline without raw attribution. **Upstream reason** shows a classified provider condition (for example rate limited, quota or credits exhausted, context window exceeded, location not supported, account verification required, or model unavailable), and **Failure detail** shows text written by the proxy, such as a stall timeout or byte-cap limit. When the provider names a published error class or finish reason (for example Gemini or Vertex `INVALID_ARGUMENT`, or `MALFORMED_FUNCTION_CALL`), Failure detail also shows that token; new rows do not store the provider's own error text. Older `usage.jsonl` files may still contain it, but the dashboard drops it when reading them. A proxy replay refusal keeps HTTP 429 and error code `upstream_reset_replay_refused`, but its request and attempt failure cause is `transport-ambiguous`, with resend permission diff --git a/gui/src/pages/logs-failure-summary.ts b/gui/src/pages/logs-failure-summary.ts index 0dbc43c29dc..b263371396b 100644 --- a/gui/src/pages/logs-failure-summary.ts +++ b/gui/src/pages/logs-failure-summary.ts @@ -157,7 +157,9 @@ export function failureDetailLabel(value: unknown, t: TFn): string | undefined { const detail = boundedFailureDetail(value); if (!detail) return undefined; if (detail.code === "provider_error") { - const label = detail.type ? t("logs.failureDetail.providerErrorTyped", { type: detail.type }) : t(FAILURE_DETAIL_KEYS.provider_error); + const typed = detail.type ? t("logs.failureDetail.providerErrorTyped", { type: detail.type }) : t(FAILURE_DETAIL_KEYS.provider_error); + // The provider's closed finishReason is a machine token, appended like the validation pairs. + const label = detail.finishReason ? `${typed} [finishReason: ${detail.finishReason}]` : typed; return detail.validation ? `${label}: ${detail.validation.map(issue => `${issue.field} [${issue.type}]`).join("; ")}` : label; } if (detail.code === "header_timeout" && detail.value !== undefined) { diff --git a/scripts/test-layout/layout-usage.json b/scripts/test-layout/layout-usage.json index 26ebfe57b7e..508b645fd67 100644 --- a/scripts/test-layout/layout-usage.json +++ b/scripts/test-layout/layout-usage.json @@ -4,6 +4,7 @@ "request-log.test.ts": "usage", "request-log-normalization.test.ts": "usage", "request-log-error-detail.test.ts": "usage", + "provider-failure-detail.test.ts": "usage", "usage-telemetry-profile-contract.test.ts": "usage", "usage-telemetry-profile-sync.test.ts": "usage", "usage-telemetry-profile-collector.test.ts": "usage", diff --git a/src/adapters/google-truncation.ts b/src/adapters/google-truncation.ts index 0e8739e5e1c..a8629fd14a2 100644 --- a/src/adapters/google-truncation.ts +++ b/src/adapters/google-truncation.ts @@ -1,4 +1,5 @@ import { redactSecretString } from "../lib/redact"; +import { withProviderFailureFacts } from "../lib/failure-detail"; import type { AdapterEvent, OcxUsage } from "../types"; /** Gemini/Vertex finishReason values that mean the turn was cut off, not cleanly stopped. */ @@ -37,10 +38,11 @@ export function vertexTruncationErrorEvent( toolCallsStarted: number, usage?: OcxUsage, ): Extract { - return { + // The request log keeps the closed finish reason that ended the turn (#8446); the wire event is unchanged. + return withProviderFailureFacts({ type: "error", message: vertexTruncationErrorMessage(finishReason), ...(usage ? { usage } : {}), ...(finishReason === "MALFORMED_FUNCTION_CALL" && toolCallsStarted === 0 ? { replaySafeBeforeOutput: true } : {}), - }; + } as const, { finishReason }); } diff --git a/src/adapters/google.ts b/src/adapters/google.ts index a449bc97946..668b66a2d7e 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -3,6 +3,7 @@ import type { AdapterFetchContext, AdapterRequest, IncomingMeta, ProviderAdapter import { debugDroppedFrame, debugProviderDiagnosticLazy } from "../lib/debug"; import { createToolCallIdAllocator } from "./tool-call-id"; import { openCodeGoGeminiEndpoint } from "../types/opencode-go"; +import { withProviderFailureFacts } from "../lib/failure-detail"; import { createImageBudget, materializeInlineImage, MAX_ENCODED_BYTES_PER_IMAGE, artifactHttpUrl } from "../images/artifacts"; import type { AdapterEvent, @@ -1393,12 +1394,14 @@ export function createGoogleAdapter(provider: OcxProviderConfig): ProviderAdapte // Inline provider error inside a 200 stream → terminal error (see openai-chat.ts). if (chunk.error) { - const err = chunk.error as { message?: string } | undefined; + const err = chunk.error as { message?: string; status?: unknown } | undefined; // Clear-on-invalid: a signature rejection means our replayed thoughtSignatures are stale. // Drop the cache entry and durable store entry for rejected calls so the next turn // starts clean instead of re-injecting a bad sig. handleSignatureRejection(err?.message); - yield { type: "error", message: err?.message ?? "upstream error" }; + // Only the closed RPC status class and finish reason ride along for the request log (#8446). + yield withProviderFailureFacts({ type: "error", message: err?.message ?? "upstream error" } as AdapterEvent, + { type: err?.status, finishReason: lastFinishReason }); return "terminate"; } diff --git a/src/adapters/upstream-http-error.ts b/src/adapters/upstream-http-error.ts index 25b49e38c4b..733ecbbc06a 100644 --- a/src/adapters/upstream-http-error.ts +++ b/src/adapters/upstream-http-error.ts @@ -1,5 +1,6 @@ import { inheritAccountAutoPauseResponse } from "../lib/account-auto-pause-response"; import { readBoundedResponseBody } from "../lib/bounded-body"; +import { providerErrorTypeOf, withProviderFailureFacts } from "../lib/failure-detail"; import { redactSecretString } from "../lib/redact"; const ABSOLUTE_PATH_PATTERN = /(?:\/Users\/[^ "';,]+|\/home\/[^ "';,]+|\/root\/[^ "';,]*|[A-Za-z]:\\Users\\[^ "';,]+)/g; @@ -41,9 +42,10 @@ export async function normalizeUpstreamHttpErrorResponse( const headers = new Headers(res.headers); headers.delete("content-encoding"); headers.delete("content-length"); - return inheritAccountAutoPauseResponse(res, new Response(await opts.formatMessage(payloadText), { + // The rewritten body is display text; keep the provider's closed error class for the request log (#8446). + return withProviderFailureFacts(inheritAccountAutoPauseResponse(res, new Response(await opts.formatMessage(payloadText), { status: res.status, statusText: res.statusText, headers, - })); + })), { type: providerErrorTypeOf(parseUpstreamJsonPayload(payloadText)) }); } diff --git a/src/lib/failure-detail.ts b/src/lib/failure-detail.ts index 6fc01807904..f7ed1ef4703 100644 --- a/src/lib/failure-detail.ts +++ b/src/lib/failure-detail.ts @@ -23,24 +23,43 @@ export const FAILURE_DETAIL_CODES = [ export type FailureDetailCode = typeof FAILURE_DETAIL_CODES[number]; -/** Closed provider error classes a provider_error detail may name (the Anthropic error envelope types). */ +/** + * Closed provider error classes a provider_error detail may name: the Anthropic error envelope types + * and the canonical Google RPC status names (google.rpc.Code) that Gemini, Vertex and Antigravity put in + * `error.status`. Both are published vocabularies, so naming one discloses nothing the provider wrote. + */ export const FAILURE_DETAIL_PROVIDER_TYPES = [ "invalid_request_error", "authentication_error", "permission_error", "not_found_error", "rate_limit_error", "api_error", "overloaded_error", "request_too_large", + "CANCELLED", "UNKNOWN", "INVALID_ARGUMENT", "DEADLINE_EXCEEDED", "NOT_FOUND", "ALREADY_EXISTS", + "PERMISSION_DENIED", "RESOURCE_EXHAUSTED", "FAILED_PRECONDITION", "ABORTED", "OUT_OF_RANGE", + "UNIMPLEMENTED", "INTERNAL", "UNAVAILABLE", "DATA_LOSS", "UNAUTHENTICATED", ] as const; export type FailureDetailProviderType = typeof FAILURE_DETAIL_PROVIDER_TYPES[number]; +/** Gemini/Vertex candidate finishReason values (the published FinishReason enum) a provider_error may name. */ +export const FAILURE_DETAIL_FINISH_REASONS = [ + "FINISH_REASON_UNSPECIFIED", "STOP", "MAX_TOKENS", "SAFETY", "RECITATION", "LANGUAGE", "OTHER", "BLOCKLIST", + "PROHIBITED_CONTENT", "SPII", "MALFORMED_FUNCTION_CALL", "IMAGE_SAFETY", "IMAGE_PROHIBITED_CONTENT", + "IMAGE_RECITATION", "IMAGE_OTHER", "NO_IMAGE", "UNEXPECTED_TOOL_CALL", "TOO_MANY_TOOL_CALLS", +] as const; + +export type FailureDetailFinishReason = typeof FAILURE_DETAIL_FINISH_REASONS[number]; + export interface FailureDetail { code: FailureDetailCode; /** Milliseconds for timeouts and stalls, bytes for an overflow. */ value?: number; type?: FailureDetailProviderType; + /** The provider's own closed finish reason for the turn that failed; provider_error only. */ + finishReason?: FailureDetailFinishReason; validation?: ValidationIssue[]; } const CODES: ReadonlySet = new Set(FAILURE_DETAIL_CODES); const TYPES: ReadonlySet = new Set(FAILURE_DETAIL_PROVIDER_TYPES); +const FINISH_REASONS: ReadonlySet = new Set(FAILURE_DETAIL_FINISH_REASONS); const MAX_VALUE = 1e12; /** Codes whose value parameter carries meaning; the value is dropped for any other code. */ @@ -61,9 +80,59 @@ export function boundedFailureDetail(value: unknown): FailureDetail | undefined const valid = typeof number === "number" && Number.isSafeInteger(number) && number >= 0 && number <= MAX_VALUE && VALUED.has(code); const type = code === "provider_error" && typeof record.type === "string" && TYPES.has(record.type) ? record.type as FailureDetailProviderType : undefined; + const finishReason = code === "provider_error" && typeof record.finishReason === "string" && FINISH_REASONS.has(record.finishReason) + ? record.finishReason as FailureDetailFinishReason : undefined; const validation = code === "provider_error" ? boundedValidationIssues(record.validation) : undefined; const keys = Object.keys(record); - const exact = keys.every(key => key === "code" || (key === "value" && valid) || (key === "type" && type !== undefined) || (key === "validation" && validation !== undefined && validation === record.validation)); + const exact = keys.every(key => key === "code" || (key === "value" && valid) || (key === "type" && type !== undefined) + || (key === "finishReason" && finishReason !== undefined) + || (key === "validation" && validation !== undefined && validation === record.validation)); if (exact) return record as unknown as FailureDetail; - return { code, ...(valid ? { value: number as number } : {}), ...(type ? { type } : {}), ...(validation ? { validation } : {}) }; + return { + code, ...(valid ? { value: number as number } : {}), ...(type ? { type } : {}), + ...(finishReason ? { finishReason } : {}), ...(validation ? { validation } : {}), + }; +} + +/** The closed facts an adapter can state about a provider failure it parsed off the wire. */ +export interface ProviderFailureFacts { + type?: FailureDetailProviderType; + finishReason?: FailureDetailFinishReason; +} + +// Keyed by the adapter event object, so the facts can never be serialized with it; an event a guard +// rebuilds simply carries none. +const providerFailureFacts = new WeakMap(); + +/** + * Attach the closed provider facts to an adapter error event. Values outside the closed sets are + * dropped here, so an adapter can pass what the provider sent without checking it first. + */ +export function withProviderFailureFacts(event: T, facts: { type?: unknown; finishReason?: unknown }): T { + const type = typeof facts.type === "string" && TYPES.has(facts.type) ? facts.type as FailureDetailProviderType : undefined; + const finishReason = typeof facts.finishReason === "string" && FINISH_REASONS.has(facts.finishReason) + ? facts.finishReason as FailureDetailFinishReason : undefined; + if (type || finishReason) providerFailureFacts.set(event, { ...(type ? { type } : {}), ...(finishReason ? { finishReason } : {}) }); + return event; +} + +export function providerFailureDetailOf(event: object): ProviderFailureFacts | undefined { + return providerFailureFacts.get(event); +} + +const isObject = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +/** + * The closed provider error class a parsed upstream error body names: the envelope `error.type`, Google's + * `error.status` (also as the first element of a streamed JSON array), or a top-level `type`. + */ +export function providerErrorTypeOf(payload: unknown): FailureDetailProviderType | undefined { + const record = Array.isArray(payload) ? payload[0] : payload; + if (!isObject(record)) return undefined; + const nested = isObject(record.error) ? record.error : undefined; + for (const value of [nested?.type, nested?.status, record.type]) { + if (typeof value === "string" && TYPES.has(value)) return value as FailureDetailProviderType; + } + return undefined; } diff --git a/src/server/chat-native.ts b/src/server/chat-native.ts index c25f4909ae2..1c038f394bf 100644 --- a/src/server/chat-native.ts +++ b/src/server/chat-native.ts @@ -1,4 +1,4 @@ -import { recordUpstreamValidationDetail } from "./request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail } from "./request-log-failure-detail"; import { normalizeNativeChatContent } from "../adapters/openai-chat/native-content"; import { attachComboFailureEvidence, ownedChatErrorPayload, type ClientErrorContract } from "../lib/client-error-body"; import type { FailureVerdictSink } from "../chat/failure-verdict"; @@ -754,7 +754,7 @@ export async function runNativeChatAttempt( let upstreamMessage: string | undefined; try { const parsedError = JSON.parse(bodyText) as Rec; - recordUpstreamValidationDetail(logCtx, response.status, parsedError); + recordUpstreamProviderErrorDetail(logCtx, response.status, parsedError); const nested = isRec(parsedError.error) ? parsedError.error : undefined; const details = nested ?? parsedError; if (typeof details.type === "string") upstreamType = details.type; diff --git a/src/server/request-log-failure-detail.ts b/src/server/request-log-failure-detail.ts index e521be7b69f..05b6829bac7 100644 --- a/src/server/request-log-failure-detail.ts +++ b/src/server/request-log-failure-detail.ts @@ -10,7 +10,7 @@ import { knownModelIdentity } from "../lib/model-identity"; * upstreamError, which internal classifiers and status heuristics read and which never reaches a row. * Only proxy-composed text may be passed here. */ -import { boundedFailureDetail, type FailureDetail, type FailureDetailCode } from "../lib/failure-detail"; +import { boundedFailureDetail, providerErrorTypeOf, providerFailureDetailOf, type FailureDetail, type FailureDetailCode } from "../lib/failure-detail"; export type { FailureDetail }; @@ -95,3 +95,61 @@ export function recordUpstreamValidationDetail(logCtx: { failureDetail?: Failure ? record.error as Record : undefined; logCtx.failureDetail = boundedFailureDetail({ code: "provider_error", type: nested?.type ?? record.type, validation }); } + +/** + * Capture the closed provider error class from a raw upstream error body (#8446): the envelope + * `type` (Anthropic/OpenAI) or Google's `error.status`, the latter also as the first element of a + * streamed JSON array. Validation pairs keep their existing capture and win when present. Only + * callers holding the provider's own body may use this; a body this proxy formatted is not evidence. + * An adapter transport that rewrote the body (normalizeUpstreamHttpErrorResponse) attached the class + * it read from the original to `response`, which is preferred. + */ +export function recordUpstreamProviderErrorDetail( + logCtx: { failureDetail?: FailureDetail }, + status: number, + payload: unknown, + response?: object, +): void { + if (status < 400 || status >= 600) return; + const before = logCtx.failureDetail; + recordUpstreamValidationDetail(logCtx, status, payload); + if (logCtx.failureDetail !== before) { providerDetails.set(logCtx, logCtx.failureDetail!); return; } + let type = response ? providerFailureDetailOf(response)?.type : undefined; + if (type === undefined) { + let parsed = payload; + if (typeof payload === "string") { + if (payload.length > 256 * 1024) return; + try { parsed = JSON.parse(payload); } catch { return; } + } + type = providerErrorTypeOf(parsed); + } + if (type === undefined) return; + const detail: FailureDetail = { code: "provider_error", type }; + logCtx.failureDetail = detail; + providerDetails.set(logCtx, detail); +} + +/** The provider detail each context last recorded, so a later success can retract exactly that one. */ +const providerDetails = new WeakMap(); + +/** + * Request-log side of every adapter event (the parse seam in request-transport.ts). An error event an + * adapter tagged with closed provider facts records them as a provider_error detail; a later `done` on + * the same request (a retry or replay that succeeded) retracts a detail this path recorded, so a + * successful row never names a provider failure it recovered from. + */ +export function observeProviderFailureEvent(logCtx: { failureDetail?: FailureDetail }, event: { type: string }): void { + if (event.type === "done") { + const recorded = providerDetails.get(logCtx); + if (recorded !== undefined && logCtx.failureDetail === recorded) delete logCtx.failureDetail; + providerDetails.delete(logCtx); + return; + } + if (event.type !== "error") return; + const facts = providerFailureDetailOf(event); + if (!facts) return; + const detail = boundedFailureDetail({ code: "provider_error", ...facts }); + if (!detail) return; + logCtx.failureDetail = detail; + providerDetails.set(logCtx, detail); +} diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 2088d3ec6ae..5954eedd158 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -1,8 +1,8 @@ -import { recordUpstreamValidationDetail } from "../request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail } from "../request-log-failure-detail"; import { clientFacingModelText } from "../../lib/error-destination"; import { responsesSpendTarget } from "./request-spend-target"; import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; -import { markProxyError, carryProxyError } from "../../lib/client-error-body"; +import { markProxyError, carryProxyError, isProxyError } from "../../lib/client-error-body"; import { codexAdapterBuildRefusal } from "./passthrough-build-error"; import { createAutoPauseRefreshErrorMapper, recoverAutoPausedAccount } from "./account-auto-pause-recovery"; import { wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; @@ -1864,7 +1864,7 @@ export async function prepareAdapterExchange( // `response.body` itself with the abort signal threaded through, and the combo // contract is that this body's getter is touched exactly once. A guard here would be // a second `.body` access for no gain, since the bounded reader owns settlement. - const failure = await consumeComboFailure(upstreamResponse, options.abortSignal, Date.now(), logCtx) + const failure = await consumeComboFailure(upstreamResponse, options.abortSignal, Date.now(), logCtx, !isProxyError(upstreamResponse)) .finally(cleanupUpstreamAbort); options.onConsumedComboFailure?.(failure); return failure.response; @@ -1879,7 +1879,8 @@ export async function prepareAdapterExchange( } finally { cleanupUpstreamAbort(); } - recordUpstreamValidationDetail(logCtx, upstreamResponse.status, errorText); + // The provider's own body (not a proxy-normalized one) may name its closed error class (#8446). + if (!isProxyError(upstreamResponse)) recordUpstreamProviderErrorDetail(logCtx, upstreamResponse.status, errorText, upstreamResponse); if (upstreamResponse.status === 413) { if (options.clientModelLabel) projectLocalModelRefusalLog(logCtx, options.clientModelLabel); return clientRequestedStream diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index d3bbe298afe..686bd3583a7 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -1,4 +1,4 @@ -import { recordUpstreamValidationDetail, type FailureDetail } from "../request-log-failure-detail"; +import { recordUpstreamProviderErrorDetail, recordUpstreamValidationDetail, type FailureDetail } from "../request-log-failure-detail"; import { carryProxyErrorPresentation } from "../../lib/client-error-body"; import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { comboFailureEvidence, carryProxyError } from "../../lib/client-error-body"; @@ -47,6 +47,8 @@ export async function consumeComboFailure( signal?: AbortSignal, now = Date.now(), logCtx?: { failureDetail?: FailureDetail; causeHint?: RequestFailureCause }, + /** True only when `response` is the provider's own HTTP answer, so its closed error class is evidence. */ + rawUpstream = false, ): Promise { // Read before the body: the marker lives on this Response object, and the failure below is // rebuilt as a new one that would otherwise lose it. @@ -85,7 +87,8 @@ export async function consumeComboFailure( && isRateLimitOrQuotaFailureMessage(quotaMessage); } if (body.displaySafe && !body.truncated) { - if (logCtx) recordUpstreamValidationDetail(logCtx, response.status, body.text); + if (logCtx && rawUpstream) recordUpstreamProviderErrorDetail(logCtx, response.status, body.text, response); + else if (logCtx) recordUpstreamValidationDetail(logCtx, response.status, body.text); // A combo child's row is attributed from the same evidence the parent uses: Meta's Muse // Code refusal is a plan restriction, not a rejected credential (#8352). if (logCtx) stampMetaMusePlanRefusal(logCtx, response.status, body.text); diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index c98ba4f0311..39b0d8951e0 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -1,5 +1,5 @@ import { publicOAuthErrorPresentation } from "../../oauth"; -import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; +import { observeProviderFailureEvent, projectLocalModelRefusalLog } from "../request-log-failure-detail"; import { markProxyError } from "../../lib/client-error-body"; import { formatClientRetryAfter } from "../../lib/client-retry-after"; import { SendBudgetExhaustedError, errorDestinationText, providerErrorDestinationLabel } from "../../lib/upstream-retry"; @@ -676,6 +676,7 @@ export async function prepareResponsesTransport( const observeEvent = (event: AdapterEvent, response: object): void => { delivery.noteAdapterEvent(); observeUsage(event, response); + observeProviderFailureEvent(logCtx, event); }; const parseStream = resolved.parseStream.bind(resolved); resolved.parseStream = async function* (...args) { diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index 561b51549ce..10724e0ec22 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -40,7 +40,9 @@ through the disk reader. A row keeps: evidence before discarding its text; `tests/usage/request-log-normalization.test.ts` covers both. - `failureDetail`, a closed `{ code, value?, type?, validation? }` from `src/lib/failure-detail.ts`: `code` is a member of `FAILURE_DETAIL_CODES`, `value` a safe integer (milliseconds or bytes) kept only for codes that take one, - and `type` a closed Anthropic error class kept only on `provider_error`. `recordLocalFailureDetail` in + `type` a closed provider error class (the Anthropic envelope types or a canonical Google RPC status + name such as `INVALID_ARGUMENT`) and `finishReason` a member of the Gemini/Vertex FinishReason enum, + both kept only on `provider_error`. `recordLocalFailureDetail` in `src/server/request-log-failure-detail.ts` takes the detail beside the English text it leaves on the context; `incompleteReasonDetail` maps the known `incomplete_details.reason` values (`max_output_tokens`, `upstream_stall_timeout`, `adapter_eof`), and an unknown reason stays on the context only. A relayed error @@ -62,6 +64,24 @@ partial bodies produce status-only diagnostics. GUI rendering. The dashboard appends the machine field/type pairs to its localized provider-error label. No upstream-authored free text is retained, and client-facing messages retain their existing policy. +A provider failure also keeps its closed class without validation pairs (#8446). +`recordUpstreamProviderErrorDetail` reads `error.type`, Google's `error.status` (an object or the first +element of a streamed array) or a top-level `type` from a raw upstream body: native Chat, the +translated adapter dispatch, and `consumeComboFailure` when adapter dispatch passes it the provider's +own response. A proxy-marked response and every other `consumeComboFailure` caller keep the +validation-only capture, because a body this proxy formatted is not provider evidence. An adapter +transport that rewrites the body for display (`normalizeUpstreamHttpErrorResponse`, used by Vertex and +Antigravity) reads the class from the original first and attaches it to the new response through +`withProviderFailureFacts` in `src/lib/failure-detail.ts`, a `WeakMap` keyed by the object, so +nothing new is serialized. The Google adapter tags a mid-stream `error` frame with its status and a +fail-closed Vertex truncation with its finish reason the same way, and the parse seam in +`src/server/responses/request-transport.ts` calls `observeProviderFailureEvent`, which records them. +A later `done` on the same request (a retry or empty-completion replay that succeeded) retracts the +detail this path recorded, so a successful row never names a failure it recovered from. The message +itself stays on the context like every other upstream text. The dashboard appends the finish reason as +a machine token. `tests/usage/provider-failure-detail.test.ts` drives the Vertex route for an HTTP 400, +a combo child, a mid-stream error frame, a truncated turn and a recovered replay. + `upstreamError`, `upstreamErrorCode`, `upstreamErrorType` and `upstreamRequestId` stay on the request context, where status heuristics and classifiers read them, and never reach a row, `usage.jsonl` or the service log. Bytes already in `usage.jsonl` from earlier builds are not rewritten; they are dropped on every diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 25ba4fd2811..6744c006765 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1624,7 +1624,7 @@ "hermes-profiles.test.ts": "clients", "hermes-profile-routes.test.ts": "clients" ,"cli-sync-expect-link.test.ts": "cli" ,"local-credentials.test.ts": "cli", "local-credential-generation.test.ts": "config", "link-connection-generation.test.ts": "clients" - ,"request-log-error-detail.test.ts": "usage", "upstream-condition.test.ts": "lib", "client-retry-after.test.ts": "lib", "client-retry-after-sites.test.ts": "server", "upstream-error-detail-boundaries.test.ts": "server", "request-log-identity-bounds.test.ts": "usage", "safe-response-headers.test.ts": "server", "turn-state-handles.test.ts": "server", "turn-state-handles-e2e.test.ts": "server", "client-error-body.test.ts": "lib", "chat-error-body-normalization.test.ts": "responses", "chat-error-collector-verdicts.test.ts": "responses", "server-chat-error-body-evidence.test.ts": "server" + ,"request-log-error-detail.test.ts": "usage", "provider-failure-detail.test.ts": "usage", "upstream-condition.test.ts": "lib", "client-retry-after.test.ts": "lib", "client-retry-after-sites.test.ts": "server", "upstream-error-detail-boundaries.test.ts": "server", "request-log-identity-bounds.test.ts": "usage", "safe-response-headers.test.ts": "server", "turn-state-handles.test.ts": "server", "turn-state-handles-e2e.test.ts": "server", "client-error-body.test.ts": "lib", "chat-error-body-normalization.test.ts": "responses", "chat-error-collector-verdicts.test.ts": "responses", "server-chat-error-body-evidence.test.ts": "server" ,"messages-error-normalization.test.ts": "claude-integration", "messages-header-normalization.test.ts": "claude-integration", "messages-lane-errors.test.ts": "claude-integration" ,"messages-error-contracts.test.ts": "claude-integration", "messages-overflow-verdicts.test.ts": "claude-integration", "messages-review-regressions.test.ts": "claude-integration", "messages-proxy-refusals.test.ts": "claude-integration", "messages-overflow-native.test.ts": "claude-integration", "messages-overflow-builtin.test.ts": "claude-integration", "messages-overflow-translated.test.ts": "claude-integration", "messages-overflow-stream.test.ts": "claude-integration", "messages-overflow-collectors.test.ts": "claude-integration", "messages-success-media-types.test.ts": "claude-integration", "messages-native-stream-parity.test.ts": "claude-integration", "messages-overflow-baseline-contract.test.ts": "claude-integration", "messages-sse-builtin-errors.test.ts": "claude-integration", diff --git a/tests/usage/provider-failure-detail.test.ts b/tests/usage/provider-failure-detail.test.ts new file mode 100644 index 00000000000..fa5c03dcc35 --- /dev/null +++ b/tests/usage/provider-failure-detail.test.ts @@ -0,0 +1,151 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { handleResponses } from "../../src/server/responses"; +import { addFinalRequestLog, type RequestLogContext, type RequestLogEntry } from "../../src/server/request-log"; +import { recordUpstreamProviderErrorDetail } from "../../src/server/request-log-failure-detail"; +import { boundedFailureDetail, providerFailureDetailOf } from "../../src/lib/failure-detail"; +import { createGoogleAdapter } from "../../src/adapters/google"; +import { failureDetailLabel } from "../../gui/src/pages/logs-failure-summary"; +import { withTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import type { AdapterEvent, OcxConfig, OcxProviderConfig } from "../../src/types"; + +const releaseSpendHome = acquireOwnedSpendHome(); +afterAll(() => releaseSpendHome()); + +// Upstream-authored text that must never reach a row: a credential-shaped token and prompt words. +const SECRET = "zq7Vb2LmN4pX9tR1sK8wD3fH6jY0cA5e"; +const MESSAGE = `Invalid value at contents[3] near "private prompt" for key ${SECRET}`; +const googleError = (status: string, code = 400) => ({ error: { code, status, message: MESSAGE } }); + +describe("closed provider failure detail contract (#8446)", () => { + test("keeps a Google RPC status class and a closed finishReason on provider_error", () => { + expect(boundedFailureDetail({ code: "provider_error", type: "INVALID_ARGUMENT" })).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(boundedFailureDetail({ code: "provider_error", type: "FAILED_PRECONDITION", finishReason: "MALFORMED_FUNCTION_CALL" })) + .toEqual({ code: "provider_error", type: "FAILED_PRECONDITION", finishReason: "MALFORMED_FUNCTION_CALL" }); + }); + + test("drops unknown statuses and reasons, free text, and a finishReason on another code", () => { + expect(boundedFailureDetail({ code: "provider_error", type: SECRET, finishReason: MESSAGE })).toEqual({ code: "provider_error" }); + expect(boundedFailureDetail({ code: "provider_error", type: "invalid_argument" })).toEqual({ code: "provider_error" }); + expect(boundedFailureDetail({ code: "stream_failed", finishReason: "SAFETY" })).toEqual({ code: "stream_failed" }); + }); + + test("the dashboard renders the status and the finish reason as machine tokens", () => { + const t = (key: string, vars?: Record) => `${key}${vars ? JSON.stringify(vars) : ""}`; + const label = failureDetailLabel({ code: "provider_error", type: "INVALID_ARGUMENT", finishReason: "MALFORMED_FUNCTION_CALL" }, t as never); + expect(label).toContain("INVALID_ARGUMENT"); + expect(label).toContain("MALFORMED_FUNCTION_CALL"); + }); +}); + +describe("raw upstream error capture", () => { + test("records the Google status class from an object or a streamed array body, never the message", () => { + for (const body of [googleError("INVALID_ARGUMENT"), [googleError("INVALID_ARGUMENT")]]) { + const ctx: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(ctx, 400, JSON.stringify(body)); + expect(ctx.failureDetail).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(JSON.stringify(ctx.failureDetail)).not.toContain(SECRET); + } + }); + + test("keeps existing closed envelope types and validation pairs, and ignores success or unknown classes", () => { + const typed: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(typed, 400, { error: { type: "invalid_request_error", message: MESSAGE } }); + expect(typed.failureDetail).toEqual({ code: "provider_error", type: "invalid_request_error" }); + const validation: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(validation, 422, { detail: [{ loc: ["body", "store"], type: "extra_forbidden", input: SECRET }] }); + expect(validation.failureDetail).toEqual({ code: "provider_error", validation: [{ field: "store", type: "extra_forbidden" }] }); + for (const [status, body] of [[200, googleError("INVALID_ARGUMENT")], [400, { error: { type: SECRET, message: MESSAGE } }], [400, MESSAGE]] as const) { + const ctx: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(ctx, status, typeof body === "string" ? body : JSON.stringify(body)); + expect(ctx.failureDetail).toBeUndefined(); + } + const server: RequestLogContext = {}; + recordUpstreamProviderErrorDetail(server, 503, JSON.stringify(googleError("UNAVAILABLE", 503))); + expect(server.failureDetail).toEqual({ code: "provider_error", type: "UNAVAILABLE" }); + }); +}); + +describe("Google adapter failure events", () => { + const vertex = { adapter: "google", baseUrl: "https://x", googleMode: "vertex" } as OcxProviderConfig; + const collect = async (chunks: unknown[]): Promise => { + const adapter = withTestTranslatorBudget(createGoogleAdapter(vertex)); + const body = chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join(""); + const events: AdapterEvent[] = []; + for await (const event of adapter.parseStream(new Response(body, { headers: { "content-type": "text/event-stream" } }))) events.push(event); + return events; + }; + + test("a mid-stream error frame carries its closed status class off the wire", async () => { + const events = await collect([googleError("FAILED_PRECONDITION")]); + const error = events.at(-1)!; + expect(error.type).toBe("error"); + expect(providerFailureDetailOf(error)).toEqual({ type: "FAILED_PRECONDITION" }); + expect(JSON.stringify(error)).not.toContain("FAILED_PRECONDITION\"}"); + }); + + test("a truncated turn carries the finishReason that ended it", async () => { + const events = await collect([{ candidates: [{ finishReason: "MALFORMED_FUNCTION_CALL" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 0 } }]); + expect(providerFailureDetailOf(events.at(-1)!)).toEqual({ finishReason: "MALFORMED_FUNCTION_CALL" }); + }); +}); + +describe("Responses rows through the Vertex route", () => { + // The Vertex transport builds its own googleapis.com URL, so the fixture answers at fetch itself: + // nothing in these cases may leave the process. + async function run(answer: () => Response, options: { config?: Partial; model?: string } = {}) { + const realFetch = globalThis.fetch; + let calls = 0; + globalThis.fetch = (async () => { calls += 1; return answer(); }) as unknown as typeof fetch; + try { + const config = { port: 0, defaultProvider: "fixture", ...options.config, providers: { fixture: { + adapter: "google", googleMode: "vertex", project: "fixture-project", location: "us-central1", + baseUrl: "https://aiplatform.googleapis.com", authMode: "key", apiKey: SECRET, + } } } as OcxConfig; + const log: RequestLogContext = { model: "gemini-3-pro", provider: "fixture" }; + const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: options.model ?? "fixture/gemini-3-pro", input: "private prompt", stream: true }) }), config, log); + await response.text().catch(() => ""); + let row: RequestLogEntry | undefined; + addFinalRequestLog("fixture", Date.now(), log, response.status === 200 ? 502 : response.status, undefined, entry => { row = entry; }); + return { log, row, calls }; + } finally { globalThis.fetch = realFetch; } + } + const sse = (...frames: unknown[]) => () => new Response(frames.map(frame => `data: ${JSON.stringify(frame)}\n\n`).join(""), + { headers: { "content-type": "text/event-stream" } }); + const malformed = sse({ candidates: [{ finishReason: "MALFORMED_FUNCTION_CALL" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 0 } }); + + test("an HTTP 400 keeps INVALID_ARGUMENT in the row without the provider message", async () => { + const { row } = await run(() => Response.json(googleError("INVALID_ARGUMENT"), { status: 400 })); + expect(row?.failureDetail).toEqual({ code: "provider_error", type: "INVALID_ARGUMENT" }); + expect(JSON.stringify(row)).not.toContain(SECRET); + expect(JSON.stringify(row)).not.toContain("private prompt"); + }); + + test("a combo child keeps the class of the provider refusal", async () => { + const { log } = await run(() => Response.json(googleError("FAILED_PRECONDITION"), { status: 400 }), { + model: "pair", config: { combos: { pair: { strategy: "fallback", targets: [{ provider: "fixture", model: "gemini-3-pro" }] } } } as Partial, + }); + expect(log.failureDetail).toEqual({ code: "provider_error", type: "FAILED_PRECONDITION" }); + }); + + test("a mid-stream error frame keeps its status class in the row", async () => { + const { row } = await run(sse(googleError("FAILED_PRECONDITION"))); + expect(row?.failureDetail).toEqual({ code: "provider_error", type: "FAILED_PRECONDITION" }); + expect(JSON.stringify(row)).not.toContain(SECRET); + }); + + test("a turn the provider truncated keeps its finishReason in the row", async () => { + const { row } = await run(malformed); + expect(row?.failureDetail).toEqual({ code: "provider_error", finishReason: "MALFORMED_FUNCTION_CALL" }); + }); + + test("a replay that then succeeds leaves no provider failure detail behind", async () => { + let served = 0; + const ok = sse({ candidates: [{ content: { role: "model", parts: [{ text: "ok" }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 1 } }); + const { log, calls } = await run(() => (++served === 1 ? malformed() : ok()), { config: { emptyCompletionRetry: true } }); + expect(calls).toBe(2); + expect(log.failureDetail).toBeUndefined(); + }); +});