From da9b34b2891e439938d9ad3ff31b98d938e9de78 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 6 Nov 2026 11:15:37 +0900 Subject: [PATCH 1/4] fix(claude): avoid retries for definite context overflow Co-authored-by: gitrvc-hub --- src/claude/outbound.ts | 3 +- src/lib/upstream-retry.ts | 67 ++++++++++++++- src/protocols/encoders/messages.ts | 10 ++- src/server/claude-messages.ts | 6 +- structure/data-planes/error-detail.md | 17 ++-- structure/data-planes/protocol-paths.md | 2 +- structure/runtime.md | 2 +- .../messages-error-normalization.test.ts | 43 +++++++++- ...essages-overflow-baseline-contract.test.ts | 7 +- tests/helpers/messages-overflow-assertions.ts | 29 +++++-- tests/lib/upstream-retry.test.ts | 84 +++++++++++++++++++ 11 files changed, 247 insertions(+), 23 deletions(-) diff --git a/src/claude/outbound.ts b/src/claude/outbound.ts index fb0f9e16fc5..1a82c6320e5 100644 --- a/src/claude/outbound.ts +++ b/src/claude/outbound.ts @@ -137,7 +137,8 @@ export function isThroughputLimitText(message: string): boolean { } /** Codes that say nothing about the cause, so the message decides (#8127). */ -const GENERIC_FAILURE_CODES: ReadonlySet = new Set(["server_error", "upstream_error", "api_error", "internal_error"]); +/** Generic 5xx responses use `upstream_server_error`; 503 uses `server_is_overloaded`. */ +const GENERIC_FAILURE_CODES: ReadonlySet = new Set(["server_error", "upstream_error", "upstream_server_error", "api_error", "internal_error"]); const NON_OVERFLOW_ERROR_TYPES: ReadonlySet = new Set(["rate_limit_error", "authentication_error", "permission_error"]); const NON_OVERFLOW_STATUSES: ReadonlySet = new Set([401, 403, 429]); /** diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 2e366828e5f..978e6e389c2 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -16,6 +16,7 @@ * the shared abort helpers from here). */ import { clearableDeadline } from "./abort"; +import { isDefiniteContextOverflow } from "./context-overflow-verdict"; import { errorDestinationLabel, errorDestinationText, type ErrorDestinationLabel } from "./error-destination"; export { errorDestinationLabel, providerErrorDestinationLabel, errorDestinationText } from "./error-destination"; import { redactSecretString } from "./redact"; @@ -721,6 +722,67 @@ export async function fetchWithResetRetry( } finally { opts.onSendsConsumed?.close?.(); } } +/** Bytes of a 5xx body read to look for a context-overflow verdict; the verdict caps its input too. */ +const CONTEXT_OVERFLOW_PEEK_BYTES = 16_384; +/** A body that has not arrived by then is not evidence; the peek must never stall the retry. */ +const CONTEXT_OVERFLOW_PEEK_MS = 250; + +/** Up to `maxBytes` of a clone of the body, or what arrived before the deadline. */ +async function peekBodyText(res: Response, maxBytes: number, deadlineMs: number, signal?: AbortSignal): Promise { + const reader = res.clone().body?.getReader(); + if (!reader) return ""; + const decoder = new TextDecoder(); + let text = ""; + let bytes = 0; + let emptyChunks = 0; + const deadlineAt = performance.now() + deadlineMs; + let timer: ReturnType | undefined; + let stop = () => {}; + const timedOut = new Promise(resolve => { + stop = () => resolve(null); + timer = setTimeout(stop, deadlineMs); + signal?.addEventListener("abort", stop, { once: true }); + if (signal?.aborted) stop(); + }); + try { + while (bytes < maxBytes) { + if (signal?.aborted || performance.now() >= deadlineAt) break; + const chunk = await Promise.race([reader.read(), timedOut]); + if (!chunk || chunk.done) break; + if (chunk.value.byteLength === 0) { + if (++emptyChunks % 64 === 0) await new Promise(resolve => setTimeout(resolve, 0)); + continue; + } + emptyChunks = 0; + const bounded = chunk.value.subarray(0, maxBytes - bytes); + bytes += bounded.byteLength; + text += decoder.decode(bounded, { stream: true }); + } + return text; + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", stop); + reader.cancel().catch(() => {}); + } +} + +/** + * True when a transient-status response's body is a definite context-overflow verdict. Peeks a + * clone within a byte and time bound, so the caller still receives the untouched body. A slow + * or endless body keeps the normal retry unless its observed prefix already proves overflow. + * "Too many tokens" alone is not enough here: a 5xx + * "Too many tokens, please wait" is a throttle that the retry exists for. + */ +async function isContextOverflowResponse(res: Response, signal?: AbortSignal): Promise { + if ((res.headers.get("content-type") ?? "").includes("text/event-stream")) return false; + try { + const text = await peekBodyText(res, CONTEXT_OVERFLOW_PEEK_BYTES, CONTEXT_OVERFLOW_PEEK_MS, signal); + return text.length > 0 && isDefiniteContextOverflow(res.status, text.replace(/too many tokens/gi, "")); + } catch { + return false; + } +} + /** * fetchWithResetRetry plus the caller-selected transient-5xx policy, PRE-STREAM only. * A received HTTP error follows that policy; an ambiguous reset's non-replayable @@ -782,10 +844,13 @@ export async function fetchWithTransientRetry( // thing about its own failure, so the ladder stops and hands the response back untouched. if (res.ok || !isTransientUpstreamStatus(res.status) || isNonReplayableResponse(res) || res.headers.get(REPLAY_REFUSAL_NO_RETRY_HEADER) === REPLAY_REFUSAL_NO_RETRY_VALUE) return res; + if (opts.abortSignal?.aborted || Date.now() - attemptStart > slowAttemptMs) return res; + // A 5xx that says the input does not fit is deterministic: resending the same body is + // refused the same way, so it goes back for the caller to classify (#8127). + if (await isContextOverflowResponse(res, opts.abortSignal)) return res; // Checked before cancelResponseBodyBestEffort so an already-aborted caller never receives // a response whose body we just cancelled. if (opts.abortSignal?.aborted) return res; - if (Date.now() - attemptStart > slowAttemptMs) return res; const instructedDelay = retryAfterDelayMs(res.headers); // The deadline is the CALLER'S, not this module's default. Reading the constant directly // broke it in both directions: a caller with a 30s budget slept the full 45s an upstream diff --git a/src/protocols/encoders/messages.ts b/src/protocols/encoders/messages.ts index 99c3326af0e..1206ed814d9 100644 --- a/src/protocols/encoders/messages.ts +++ b/src/protocols/encoders/messages.ts @@ -408,12 +408,14 @@ function createAnthropicMessageWriter( } const message = typeof error.message === "string" ? error.message : "upstream request failed"; const deterministic = error.code === UNDECLARED_TOOL_CALL_ERROR_CODE; - const overflow = isContextOverflowError(error, message); + const transportFailure = isEncoderTransportFailure(terminal); + const classified = transportFailure ? { ...error, code: "upstream_stream_error", status: 502 } : error; + const overflow = isContextOverflowError(classified, message); const original = encoderOriginalErrorEvidence(terminal); - fail(anthropicFailedStatus(error, message), message, !deterministic, + fail(anthropicFailedStatus(classified, message), message, !deterministic, deterministic ? error.code as string : overflow ? "context_length_exceeded" : undefined, - hasProxyErrorMessage(options.proxyErrorContext, message, { type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined }), isEncoderTransportFailure(terminal) ? "transport_failed" : undefined, - { ...(original ?? { message, type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined }), status: original?.status ?? anthropicFailedStatus(error, message) }); + hasProxyErrorMessage(options.proxyErrorContext, message, { type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined }), transportFailure ? "transport_failed" : undefined, + { ...(original ?? { message, type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined }), status: original?.status ?? anthropicFailedStatus(classified, message) }); }, overflow() { fail(413, "upstream translation buffer exceeded the safe limit", false, "translation_buffer_limit", true); diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index 37f9523613b..710f642df6c 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -1338,7 +1338,11 @@ async function handleClaudeMessagesWithBudget( || (![401, 403, 429].includes(response.status) && Boolean(nested || flat) && isContextOverflowError(errorObject, message)); } catch { - if (text) message = `upstream error (${response.status}): ${text.slice(0, 400)}`; + // A plain-text gateway body carries the same verdict with no envelope to read a code from. + const plain = text.trim().slice(0, 400); + contextError = Boolean(plain) && ![401, 402, 403, 429].includes(response.status) + && isContextOverflowError({}, plain); + if (text) message = contextError ? plain : `upstream error (${response.status}): ${text.slice(0, 400)}`; } } catch { /* keep fallback message */ } const upstreamRetryAfter = response.headers.get("retry-after"); diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index c6873ae02b4..9938b6d58b5 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -207,7 +207,14 @@ rows to client or catalog model identities and validated provider ids, omitting is interpolated into those local messages. No serialized flag, code or header mints identity. The shared builders live in `src/combos/resolve.ts`, `src/server/admission-model-scope.ts`, `src/server/responses/input-admission.ts` and `src/oauth/generic-account-failover.ts`. -`src/lib/upstream-retry.ts` owns the retry constructors. +`src/lib/upstream-retry.ts` owns the retry constructors. For a transient 5xx, `fetchWithTransientRetry` +does not resend a response whose bounded body prefix proves context overflow. It reads at most 16,384 +bytes from a clone within 250ms or until cancellation, leaving the returned body intact. Event streams +and bodies without an observed verdict retain the normal retry; a verdict observed before a slow or +endless body stalls still stops retry. A throttle worded only as "too many tokens" is not a verdict. +Body inspection time does not count toward the upstream slow-attempt budget, and empty chunks cannot +starve the deadline or cancellation. The retry and cancellation +contracts are exercised in `tests/lib/upstream-retry.test.ts`. `InvalidSendBudgetError` displays only a finite non-negative number or `invalid`. `UpstreamRetryEvidenceError` uses fixed retry-evidence wording and retains the original rejection only as `Error.cause`. Chat request-read failures keep typed local size/encoding messages; other @@ -368,11 +375,11 @@ formatter message before selecting the fixed incomplete or compaction sentence. Internal overflow decisions and HTTP status stay separate from this client-message check: a coded raw-collector throughput refusal still answers 400 without -selecting the compaction sentence. The shared `isContextOverflowError` source and its #8128 suite -remain unchanged; `tests/claude-integration/messages-overflow-baseline-contract.test.ts` binds their -source bytes. The executable overflow golden compares actual client recovery-phrase recognition, +selecting the compaction sentence. The shared `isContextOverflowError` adds `upstream_server_error` +to the generic codes; its remaining source and the #8128 suite stay byte-identical, enforced by +`tests/claude-integration/messages-overflow-baseline-contract.test.ts`. The executable overflow golden compares actual client recovery-phrase recognition, HTTP status and the original internal decisions independently on every fixture/path row. -The direct Messages writer also applies that shared verdict to the emitted code: a definite generic-coded overflow carries `context_length_exceeded`, the fixed compaction sentence and HTTP 400 on collection, even with `retryable: true`; original error evidence and proxy/transport metadata remain separate. `tests/helpers/messages-overflow-assertions.ts` records the five affected fixture ids only on direct-stream/direct-fold as explicit expected changes to the immutable golden. All other paths keep their baseline comparisons; neither the fixture data nor the shared predicate is regenerated. +The direct Messages writer also applies that shared verdict to the emitted code: a definite generic-coded overflow carries `context_length_exceeded`, the fixed compaction sentence and HTTP 400 on collection, even with `retryable: true`; original error evidence and proxy/transport metadata remain separate. `tests/helpers/messages-overflow-assertions.ts` records the affected fixture ids only on direct-stream/direct-fold as explicit expected changes to the immutable golden. All other paths keep their baseline comparisons; the fixture data is not regenerated. The direct encoder retains original adapter fields by terminal identity independently of shared classification. Buffered passthrough and count_tokens preserve the upstream status and closed type/code. The type chosen diff --git a/structure/data-planes/protocol-paths.md b/structure/data-planes/protocol-paths.md index 732ff301a4b..d55a6db29a5 100644 --- a/structure/data-planes/protocol-paths.md +++ b/structure/data-planes/protocol-paths.md @@ -227,7 +227,7 @@ converters' own helpers, exported from `src/chat/outbound.ts` and `src/claude/ou `chatCompletionsFailedResponse`, `chatCompletionsIncompleteOutcome`, `anthropicIncompleteOutcome`, `anthropicFailedStatus`, `isContextOverflowError`, the message snapshot, the web-search pair), so the frames a client receives are the converters' frames. A fold is the encoded stream read by -the existing collector. Messages terminal errors use the shared overflow verdict for their code as well as their status: definite generic-coded refusals carry `context_length_exceeded` even when marked retryable; the deterministic undeclared-tool refusal keeps its code. `tests/responses/protocol-direct-encoders-messages.test.ts` checks stream/fold parity for all four generic codes, plus an `upstream_error` type variant. The original adapter evidence and proxy/transport metadata still reach the [owned Messages formatter](error-detail.md#messages-error-bodies-and-headers). Only producer-marked watchdog/EOF terminals retain local Chat wording; upstream copies of their reason or message follow [client error normalization](error-detail.md#client-error-bodies-chat). Explicit incomplete reasons use the [shared normalization contract](../transports/responses-wire-shapes.md#adapter-incomplete-reasons) before the writers classify the outcome. +the existing collector. Messages terminal errors use the shared overflow verdict for their code as well as their status: definite generic-coded refusals carry `context_length_exceeded` even when marked retryable; the deterministic undeclared-tool refusal keeps its code. Producer-marked reader exceptions retain transport classification even when their text contains overflow wording; `tests/claude-integration/messages-error-normalization.test.ts` checks that they do not acquire a context-overflow code. `tests/responses/protocol-direct-encoders-messages.test.ts` checks stream/fold parity for all four generic codes, plus an `upstream_error` type variant. The original adapter evidence and proxy/transport metadata still reach the [owned Messages formatter](error-detail.md#messages-error-bodies-and-headers). Only producer-marked watchdog/EOF terminals retain local Chat wording; upstream copies of their reason or message follow [client error normalization](error-detail.md#client-error-bodies-chat). Explicit incomplete reasons use the [shared normalization contract](../transports/responses-wire-shapes.md#adapter-incomplete-reasons) before the writers classify the outcome. Deliberate equivalences, pinned by the parity tests: a Chat function call is delivered as one complete tool-call chunk when it completes, as the converter always did; Messages streams diff --git a/structure/runtime.md b/structure/runtime.md index 0ab790f55b5..7dd830ad2bb 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -1,6 +1,6 @@ # Runtime -The minute sweep checks persisted activation deadlines locally; only missing deadlines trigger metadata discovery. See the [quota activation contract](providers/openai-tiers.md#public-provider-contract). Codex control-plane ids are defined in the dependency-free `src/codex/control-plane-models.ts`; `src/router.ts` routes these exact ids unchanged to canonical OpenAI, retaining the existing error when no OpenAI provider is enabled. They do not widen the public native roster. Catalog requests and scheduled source refreshes follow the [nonblocking catalog observation contract](catalog.md#shared-catalog). Async runtime persistence awaits the fresh [selection admission](codex-cli-update.md#worker-realm-and-the-selection-commit-window), rechecks observed selection/epoch and caller-currentness changes after that await, and returns detached frozen results; signal cancellation makes the persistence wrapper return null without writing, while raw async resolver rejections and exceptions without signal cancellation retain their rejection behavior; bounded exec rejects promptly and releases output readers on deadline or abort, independently awaits actual child exit, and escalates SIGTERM to SIGKILL after the shared 2-second grace (Windows termination is forceful). Known Codex.app and ChatGPT.app commands at any location or alias follow the [execution trust contract](clients/chatgpt-desktop.md#codex-app-execution-trust); automatic discovery also requires matching bundle identity; the existing ChatGPT bundle trust policy (safe ownership, permissions, non-symlink paths, and OpenAI signatures for both the bundle and the exact executable) is enforced at every execution boundary in src/codex/exec-boundary.ts before the resolved canonical target is spawned, not once at selection. Version probes retain that execution's bundle classification for selection, so a bare PATH alias of an app cannot supersede a discovered persisted runtime; selection does not resolve PATH again after the probe. Stored and displayed command spelling remains unchanged, and this observation grants no trust to a later execution. The configured-native Claude overflow rewrite retains its existing 64 Ki-character classification and count scans; the final Messages SSE boundary then applies [error-response normalization](data-planes/error-detail.md), preserving client recovery decisions within its bound. Internal consumers retain the original rewrite and accounting contracts (`tests/claude-integration/claude-prompt-too-long.test.ts`). On the translated Messages-to-Responses lane, `isContextOverflowError` in `src/claude/outbound.ts` decides overflow for the streamed `response.failed` handler, the non-stream error branch and the failed-JSON branch: a coded `context_length_exceeded` always counts; an uncoded or generic-coded (`server_error`, `upstream_error`, `api_error`, `internal_error`) refusal counts only on wording that asserts the request exceeds a size limit, never with a 401/403/429 status, a rate-limit or credential type, throughput wording or another specific code, and then answers 400 `prompt is too long` so Claude Code compacts (#8127, `tests/claude-integration/claude-uncoded-context-overflow.test.ts`). Optional native desktop observation follows the [Codex Desktop startup gate and awaited teardown](clients/codex-desktop.md#proxy-startup-preference). Claude Code tier defaults are projected by `src/claude/model-default-settings.ts` into user settings only after explicit preview/apply, independently of launcher environment injection; see the [management contract](gui-and-management-api.md). The explicit projection pairs selected aliases with readable labels from `src/claude/context-windows.ts`, preserving custom labels and clearing only a label matching its current alias when an explicitly selected tier becomes a non-alias; unset tiers retain both settings model IDs and labels. No startup or periodic sync writes these defaults. +The minute sweep checks persisted activation deadlines locally; only missing deadlines trigger metadata discovery. See the [quota activation contract](providers/openai-tiers.md#public-provider-contract). Codex control-plane ids are defined in the dependency-free `src/codex/control-plane-models.ts`; `src/router.ts` routes these exact ids unchanged to canonical OpenAI, retaining the existing error when no OpenAI provider is enabled. They do not widen the public native roster. Catalog requests and scheduled source refreshes follow the [nonblocking catalog observation contract](catalog.md#shared-catalog). Async runtime persistence awaits the fresh [selection admission](codex-cli-update.md#worker-realm-and-the-selection-commit-window), rechecks observed selection/epoch and caller-currentness changes after that await, and returns detached frozen results; signal cancellation makes the persistence wrapper return null without writing, while raw async resolver rejections and exceptions without signal cancellation retain their rejection behavior; bounded exec rejects promptly and releases output readers on deadline or abort, independently awaits actual child exit, and escalates SIGTERM to SIGKILL after the shared 2-second grace (Windows termination is forceful). Known Codex.app and ChatGPT.app commands at any location or alias follow the [execution trust contract](clients/chatgpt-desktop.md#codex-app-execution-trust); automatic discovery also requires matching bundle identity; the existing ChatGPT bundle trust policy (safe ownership, permissions, non-symlink paths, and OpenAI signatures for both the bundle and the exact executable) is enforced at every execution boundary in src/codex/exec-boundary.ts before the resolved canonical target is spawned, not once at selection. Version probes retain that execution's bundle classification for selection, so a bare PATH alias of an app cannot supersede a discovered persisted runtime; selection does not resolve PATH again after the probe. Stored and displayed command spelling remains unchanged, and this observation grants no trust to a later execution. The configured-native Claude overflow rewrite retains its existing 64 Ki-character classification and count scans; the final Messages SSE boundary then applies [error-response normalization](data-planes/error-detail.md), preserving client recovery decisions within its bound. Internal consumers retain the original rewrite and accounting contracts (`tests/claude-integration/claude-prompt-too-long.test.ts`). On the translated Messages-to-Responses lane, `isContextOverflowError` in `src/claude/outbound.ts` decides overflow for the streamed `response.failed` handler, the non-stream error branch and the failed-JSON branch: a coded `context_length_exceeded` always counts; an uncoded or generic-coded (`server_error`, `upstream_error`, `upstream_server_error`, `api_error`, `internal_error`) refusal counts only on wording that asserts the request exceeds a size limit, never with a 401/403/429 status, a rate-limit or credential type, throughput wording or another specific code, and then answers 400 `prompt is too long` so Claude Code compacts (#8127, `tests/claude-integration/claude-uncoded-context-overflow.test.ts`). Generic 5xx responses use `upstream_server_error`, while 503 uses `server_is_overloaded`. Plain-text non-2xx overflow is normalized on the translated Messages path; HTTP 401, 402, 403 and 429 retain their original status. The pre-stream transient retry stops on a definite overflow observed within its bounded body peek, so the same request is not resent. Optional native desktop observation follows the [Codex Desktop startup gate and awaited teardown](clients/codex-desktop.md#proxy-startup-preference). Claude Code tier defaults are projected by `src/claude/model-default-settings.ts` into user settings only after explicit preview/apply, independently of launcher environment injection; see the [management contract](gui-and-management-api.md). The explicit projection pairs selected aliases with readable labels from `src/claude/context-windows.ts`, preserving custom labels and clearing only a label matching its current alias when an explicitly selected tier becomes a non-alias; unset tiers retain both settings model IDs and labels. No startup or periodic sync writes these defaults. ## Bun runtime and test runner diff --git a/tests/claude-integration/messages-error-normalization.test.ts b/tests/claude-integration/messages-error-normalization.test.ts index 02e5eadb6ad..3c833e2e379 100644 --- a/tests/claude-integration/messages-error-normalization.test.ts +++ b/tests/claude-integration/messages-error-normalization.test.ts @@ -3,6 +3,7 @@ import { handleClaudeCountTokens, handleClaudeMessages } from "../../src/server/ import { collectAnthropicMessage, responsesSseToAnthropicSse } from "../../src/claude/outbound"; import { foldAnthropicMessage, encodeAnthropicMessageSse } from "../../src/protocols/encoders/messages"; import { createTestTranslatorBudget } from "../helpers/translator-budget"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import type { AdapterEvent, OcxConfig } from "../../src/types"; const sentinel = "zq7Vb2LmN4pX9tR1sK8wD3fH6jY0cA5e"; @@ -51,6 +52,7 @@ const failures = [ { message: `Your input exceeds the context window of this model ${sentinel}`, code: "server_error" }, { message: `too many tokens per minute ${sentinel}`, type: "rate_limit_error", code: "rate_limit_exceeded", status: 429 }, { message: sentinel, type: "rate_limit_error" }, + { message: `Your input exceeds the context window of this model ${sentinel}`, code: "upstream_server_error" }, ]; for (const [index, error] of failures.entries()) { for (const direct of [false, true]) test(`${direct ? "direct" : "converter"} failed ${index} owns message and preserves decided type`, async () => { @@ -63,13 +65,13 @@ for (const [index, error] of failures.entries()) { contained(text); expect(text).toContain("event: error\n"); const payload = JSON.parse(text.split("data: ").at(-1)!); - if (index === 2 || index === 3) expect(payload.error).toMatchObject({ type: "invalid_request_error", code: "context_length_exceeded", message: "prompt is too long: The request exceeds the model's context window." }); + if (index === 2 || index === 3 || index === 6) expect(payload.error).toMatchObject({ type: "invalid_request_error", code: "context_length_exceeded", message: "prompt is too long: The request exceeds the model's context window." }); if (index === 4 || index === 5) expect(payload.error.type).toBe("rate_limit_error"); const foldBudget = createTestTranslatorBudget(); const fold = direct ? await foldAnthropicMessage((async function* () { yield { ...error, type: "error", errorType: error.type } as AdapterEvent; })(), { model: "fixture", translatorBudget: foldBudget }) : null; if (fold) { const body = await fold.text(); contained(body); - expect(fold.status).toBe(index === 2 || index === 3 ? 400 : 502); + expect(fold.status).toBe(index === 2 || index === 3 || index === 6 ? 400 : 502); expect(JSON.parse(body).error.type).toBe(payload.error.type); } }); @@ -103,6 +105,43 @@ test("direct reader exception uses the transport contract", async () => { expect(text.length).toBeGreaterThan(0); expect(text).not.toContain(sentinel); expect(text).toContain("The proxy could not reach or read the provider."); }); +test("a direct reader exception quoting overflow wording does not gain an overflow code", async () => { + const events = (async function* (): AsyncGenerator { throw new Error("prompt is too long"); })(); + const text = await new Response(encodeAnthropicMessageSse(events, { model: "fixture", translatorBudget: createTestTranslatorBudget() })).text(); + expect(JSON.parse(text.split("data: ").at(-1)!).error.type).toBe("overloaded_error"); + expect(text).not.toContain("context_length_exceeded"); +}); + +for (const [status, message, expectedStatus, expectedCode] of [ + [500, "Your input exceeds the context window of this model", 400, "context_length_exceeded"], + [402, "Your input exceeds the context window of this model", 402, undefined], + [429, "Your input exceeds the context window of this model", 429, undefined], +] as const) { + test(`translated plain-text HTTP ${status} keeps the correct overflow or billing class`, async () => { + const releaseSpend = acquireOwnedSpendHome(); + let sends = 0; + const config = { defaultProvider: "a", providers: { a: { + adapter: "openai-responses", authMode: "key", baseUrl: "https://responses.example/v1", apiKey: "key-a", + models: ["m"], transientRetryOn5xx: { attempts: 3 }, + } } } as unknown as OcxConfig; + const req = new Request("http://localhost/v1/messages", { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...input, model: "m" }) }); + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => { sends += 1; return new Response(message, { status }); }) as typeof fetch; + try { + const response = await handleClaudeMessages(req, config, { model: "", provider: "" }); + const error = (await response.json() as { error: { code?: string; message: string } }).error; + expect(response.status, JSON.stringify({ sends, error })).toBe(expectedStatus); + if (expectedCode) { + expect(sends).toBe(1); + expect(error.code).toBe(expectedCode); + expect(error.message).toBe("prompt is too long: The request exceeds the model's context window."); + } else { + expect(error.code).not.toBe("context_length_exceeded"); + } + } finally { globalThis.fetch = originalFetch; releaseSpend(); } + }); +} for (const count of [false, true]) test(`success ${count ? "count_tokens" : "Messages"} has a literal JSON media type`, async () => { const raw = count ? '{"input_tokens":42}' : '{"type":"message","content":[]}'; diff --git a/tests/claude-integration/messages-overflow-baseline-contract.test.ts b/tests/claude-integration/messages-overflow-baseline-contract.test.ts index da548858dcd..5bcc9c74dd0 100644 --- a/tests/claude-integration/messages-overflow-baseline-contract.test.ts +++ b/tests/claude-integration/messages-overflow-baseline-contract.test.ts @@ -4,12 +4,15 @@ import { readFileSync } from "node:fs"; import { repoPath } from "../helpers/repo-root"; const digest = (text: string | Uint8Array) => createHash("sha256").update(text).digest("hex"); -test("#8128 shared overflow predicate retains origin/dev source", () => { +test("#8128 shared overflow predicate changes only by adding the generic 5xx code", () => { const source = readFileSync(repoPath("src/claude/outbound.ts"), "utf8"); const start = source.indexOf("const GENERIC_FAILURE_CODES"); const end = source.indexOf("export function anthropicErrorBody", start); expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); - expect(digest(source.slice(start, end))).toBe("d1d3fd370bcc625c9638f63ff57498456e66b58cf1e258be028bc4d3318493fa"); + const predicate = source.slice(start, end); + const addition = '"upstream_server_error", '; + expect(predicate.split(addition)).toHaveLength(2); + expect(digest(predicate.replace(addition, ""))).toBe("d1d3fd370bcc625c9638f63ff57498456e66b58cf1e258be028bc4d3318493fa"); }); test("#8128 behavior suite retains origin/dev source", () => { const source = readFileSync(repoPath("tests/claude-integration/claude-uncoded-context-overflow.test.ts")); diff --git a/tests/helpers/messages-overflow-assertions.ts b/tests/helpers/messages-overflow-assertions.ts index 257a3da9451..d7e159efe3b 100644 --- a/tests/helpers/messages-overflow-assertions.ts +++ b/tests/helpers/messages-overflow-assertions.ts @@ -32,6 +32,11 @@ const DIRECT_OVERFLOW_BASELINE_RECOVERY = new Map([ ["sweep-code-upstream_error-phrase", true], ]); +// Reviewed #8129 deltas: the generic 5xx code and the translated plain-text verdict. +// These are fixture identities, not classifications derived from the implementation. +const GENERIC_5XX_LANES = new Set(["converter", "direct-stream", "direct-fold", "translated-http", "translated-fold", "translated-json"]); +const GENERIC_5XX_DIRECT_IDS = new Set(["shared-8128-12", "shared-8128-13"]); + function assertRow(base: Pick, current: Row) { const outcome = golden.outcomes[key(base)]!; expect(outcome).toBeDefined(); @@ -41,14 +46,28 @@ function assertRow(base: Pick, current: Row) { mediaType: golden.mediaTypes[outcome[7]], clientError: outcome[8], bodySha256: outcome[9], bodyLength: outcome[10] }; const correctedOverflow = DIRECT_OVERFLOW_BASELINE_RECOVERY.has(base.input.id) && (base.lane === "direct-stream" || base.lane === "direct-fold"); - const expected = correctedOverflow ? { + const directGeneric = GENERIC_5XX_DIRECT_IDS.has(base.input.id) && (base.lane === "direct-stream" || base.lane === "direct-fold"); + const genericCode = base.input.id === "sweep-code-upstream_server_error-phrase" && GENERIC_5XX_LANES.has(base.lane); + const plainText = base.input.id === "shape-plain-text-400" && (base.lane === "translated-http" || base.lane === "translated-json"); + const correctedGeneric = directGeneric || genericCode || plainText; + if (correctedGeneric) { + const wasFold = base.lane === "direct-fold"; + expect(before.status).toBe(wasFold || (genericCode && base.lane === "translated-json") ? 502 : ["translated-http", "translated-json", "translated-fold"].includes(base.lane) ? 400 : 200); + expect(before.type).toBe(directGeneric ? "overloaded_error" : genericCode && base.lane === "translated-json" ? "api_error" : "invalid_request_error"); + expect(before.code).toBeUndefined(); + expect(before.clientRecognizes).toBe(directGeneric ? base.input.id === "shared-8128-13" : true); + expect(before.internalDecisions).toEqual(base.lane === "translated-json" || base.lane === "translated-http" ? { writer: false } + : wasFold ? { status: false, writer: false, "direct-collector": false } : { status: false, writer: false }); + } + const expected = correctedOverflow || correctedGeneric ? { ...before, - status: base.lane === "direct-fold" ? 400 : 200, + status: ["direct-fold", "translated-http", "translated-fold", "translated-json"].includes(base.lane) ? 400 : 200, clientRecognizes: true, + type: "invalid_request_error", code: "context_length_exceeded", internalDecisions: base.lane === "direct-fold" ? { status: true, writer: true, "direct-collector": true } - : { status: true, writer: true }, + : base.lane === "translated-json" || base.lane === "translated-http" ? { writer: true } : { status: true, writer: true }, } : before; if (correctedOverflow) { // Keep the original observation explicit; an unrelated baseline change is not a delta. @@ -77,7 +96,7 @@ function assertRow(base: Pick, current: Row) { if (!base.input.fault) expect(base.input.eventLanes).toContain(base.lane); } expect(before.clientRecognizes).toBe(base.input.recovery[base.lane]); - expect(after.clientRecognizes).toBe(correctedOverflow ? true : base.input.recovery[base.lane]); + expect(after.clientRecognizes).toBe(correctedOverflow || correctedGeneric ? true : base.input.recovery[base.lane]); expect(after.clientRecognizes).toBe(expected.clientRecognizes); expect(after.internalDecisions).toEqual(expected.internalDecisions); const passthrough = base.lane === "passthrough" || base.lane === "count_tokens"; @@ -98,7 +117,7 @@ function assertRow(base: Pick, current: Row) { expect(after.envelopeType).toBe("error"); if (before.clientError) expect(after.compactionSentence).toBe(expected.clientRecognizes); expect(ANTHROPIC_ERROR_TYPES.has(after.type ?? "")).toBe(true); - if (ANTHROPIC_ERROR_TYPES.has(before.type ?? "")) expect(after.type).toBe(before.type); + if (ANTHROPIC_ERROR_TYPES.has(expected.type ?? "")) expect(after.type).toBe(expected.type); expect(after.code).toBe(typeof expected.code === "string" && expected.code !== "thread_unsupported_request" && ANTHROPIC_ERROR_CODES.has(expected.code) ? expected.code : undefined); } diff --git a/tests/lib/upstream-retry.test.ts b/tests/lib/upstream-retry.test.ts index 6fae42ccfef..d43d8b14d0a 100644 --- a/tests/lib/upstream-retry.test.ts +++ b/tests/lib/upstream-retry.test.ts @@ -321,6 +321,90 @@ describe("retryBackoffDelayMs", () => { expect(mock.calls).toHaveLength(1); }); + test("a 5xx context-overflow verdict is returned without a resend (#8127)", async () => { + silenceWarn(); + for (const body of [ + JSON.stringify({ error: { type: "server_error", message: "Your input exceeds the context window of this model" } }), + "Your input exceeds the context window of this model", + ]) { + const upstream = new Response(body, { status: 500 }); + const mock = mockDoFetch([upstream, new Response("duplicate"), new Response("duplicate")]); + const response = await fetchWithTransientRetry(mock.doFetch); + expect(response).toBe(upstream); + expect(await response.text()).toBe(body); + expect(mock.calls).toHaveLength(1); + } + }); + + test("the overflow peek stops at the byte limit even for multibyte text", async () => { + silenceWarn(); + const body = "é".repeat(8_192) + " Your input exceeds the context window of this model"; + const mock = mockDoFetch([new Response(body, { status: 500 }), new Response("ok")]); + expect((await fetchWithTransientRetry(mock.doFetch)).status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + + test("time spent inspecting a response body does not count as a slow upstream attempt", async () => { + silenceWarn(); + const body = new ReadableStream({ + start(controller) { setTimeout(() => { controller.enqueue(new TextEncoder().encode("unavailable")); controller.close(); }, 180); }, + }); + const mock = mockDoFetch([new Response(body, { status: 500 }), new Response("ok")]); + expect((await fetchWithTransientRetry(mock.doFetch, { slowAttemptMs: 100 })).status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + + test("aborting during an overflow peek returns the original response promptly", async () => { + silenceWarn(); + const abort = new AbortController(); + const first = new Response(new ReadableStream({}), { status: 500 }); + const mock = mockDoFetch([first, new Response("ok")]); + setTimeout(() => abort.abort(), 20); + const result = await Promise.race([ + fetchWithTransientRetry(mock.doFetch, { abortSignal: abort.signal }), + new Promise<"late">(resolve => setTimeout(() => resolve("late"), 150)), + ]); + expect(result).toBe(first); + expect(mock.calls).toHaveLength(1); + }); + + test("empty body chunks cannot starve cancellation during the overflow peek", async () => { + silenceWarn(); + const abort = new AbortController(); + let pulls = 0; + const first = new Response(new ReadableStream({ + pull(controller) { + pulls += 1; + if (pulls >= 10_000) controller.close(); else controller.enqueue(new Uint8Array()); + }, + }, { highWaterMark: 0 }), { status: 500 }); + const mock = mockDoFetch([first, new Response("ok")]); + const clone = first.clone.bind(first); + first.clone = () => { setTimeout(() => abort.abort(), 30); return clone(); }; + expect(await fetchWithTransientRetry(mock.doFetch, { abortSignal: abort.signal })).toBe(first); + expect(pulls).toBeLessThan(10_000); + expect(mock.calls).toHaveLength(1); + }); + + test("a 5xx body that never ends does not stall the overflow check (#8127)", async () => { + silenceWarn(); + const mock = mockDoFetch([new Response(new ReadableStream(), { status: 503 }), new Response("ok")]); + const response = await fetchWithTransientRetry(mock.doFetch); + expect(response.status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + + test("a 5xx throttle worded as too many tokens is still retried (#8127)", async () => { + silenceWarn(); + const mock = mockDoFetch([ + new Response(JSON.stringify({ error: { message: "Too many tokens, please wait before trying again." } }), { status: 500 }), + new Response("ok"), + ]); + const response = await fetchWithTransientRetry(mock.doFetch); + expect(response.status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + test("an instruction inside the wait deadline is still honoured before retrying (#4546)", async () => { silenceWarn(); const limited = new Response("overloaded", { From 0ded166dd6486c85a3c2f3838a283bdcbe7d5daa Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 6 Nov 2026 11:59:44 +0900 Subject: [PATCH 2/4] fix(retry): reject negated overflow verdicts --- src/lib/context-overflow-verdict.ts | 5 ++-- src/lib/upstream-retry.ts | 7 ++++- structure/data-planes/error-detail.md | 4 ++- tests/lib/upstream-retry.test.ts | 38 +++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/lib/context-overflow-verdict.ts b/src/lib/context-overflow-verdict.ts index 267c534b740..15897bdb030 100644 --- a/src/lib/context-overflow-verdict.ts +++ b/src/lib/context-overflow-verdict.ts @@ -83,9 +83,10 @@ function isDefiniteContextOverflowMessage(text: string): boolean { * arbitrary field that happens to sit in the first 500 bytes authorize a hop, so it fails closed. * * Only the exact proxy wrapper is unwrapped, within a fixed envelope budget and 16,384 characters. + * An optional caller filter can narrow the verdict on the unwrapped leaf, never broaden it. * Exported for the sidecar loops, which classify their own provider failures with the same rule. */ -export function isDefiniteContextOverflow(status: number, message: string): boolean { +export function isDefiniteContextOverflow(status: number, message: string, acceptMessage?: (text: string) => boolean): boolean { if (!CONTEXT_VERDICT_STATUSES.has(status) && status < 500) return false; if (message.length > 16_384) return false; let text = message.trim(); @@ -93,7 +94,7 @@ export function isDefiniteContextOverflow(status: number, message: string): bool for (let unwrapped = 0; unwrapped <= MAX_CONTEXT_OVERFLOW_ENVELOPES; unwrapped += 1) { const providerPrefix = /^Provider error \d{3}:\s*/.exec(text); if (providerPrefix) text = text.slice(providerPrefix[0].length).trim(); - if (!text.startsWith("{")) return isDefiniteContextOverflowMessage(text); + if (!text.startsWith("{")) return (acceptMessage?.(text) ?? true) && isDefiniteContextOverflowMessage(text); if (unwrapped === MAX_CONTEXT_OVERFLOW_ENVELOPES) return false; let payload: unknown; try { payload = JSON.parse(text); } catch { return false; } diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 978e6e389c2..55e6357d268 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -726,6 +726,10 @@ export async function fetchWithResetRetry( const CONTEXT_OVERFLOW_PEEK_BYTES = 16_384; /** A body that has not arrived by then is not evidence; the peek must never stall the retry. */ const CONTEXT_OVERFLOW_PEEK_MS = 250; +// A hypothetical, negated, or configuration-failure mention does not justify suppressing a retry. +// Inspect the unwrapped provider message, not its JSON spelling or unrelated envelope fields. +const NON_VERDICT_CONTEXT = /(?:\b(?:not|never|no|neither|nor|if|unless|whether)\b|n['\u2019]t\b)[^.!?\n]{0,100}\b(?:context(?:\s+(?:window|length))?|tokens?|(?:input|prompt)\s+is\s+too\s+long)\b|\b(?:context\s+(?:window|length)|token\s+limit)\b[^.!?\n]{0,40}\?/i; +const CONTEXT_CONFIG_FAILURE = /\b(?:fail(?:ed|ure)?|unable|cannot|error)\b[^.!?\n]{0,100}\b(?:configuration|settings|documentation)\b|\b(?:configuration|settings|documentation)\b[^.!?\n]{0,100}\b(?:fail(?:ed|ure)?|unavailable|missing|error)\b/i; /** Up to `maxBytes` of a clone of the body, or what arrived before the deadline. */ async function peekBodyText(res: Response, maxBytes: number, deadlineMs: number, signal?: AbortSignal): Promise { @@ -777,7 +781,8 @@ async function isContextOverflowResponse(res: Response, signal?: AbortSignal): P if ((res.headers.get("content-type") ?? "").includes("text/event-stream")) return false; try { const text = await peekBodyText(res, CONTEXT_OVERFLOW_PEEK_BYTES, CONTEXT_OVERFLOW_PEEK_MS, signal); - return text.length > 0 && isDefiniteContextOverflow(res.status, text.replace(/too many tokens/gi, "")); + return text.length > 0 && isDefiniteContextOverflow(res.status, text.replace(/too many tokens/gi, ""), + message => !NON_VERDICT_CONTEXT.test(message) && !CONTEXT_CONFIG_FAILURE.test(message)); } catch { return false; } diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index 9938b6d58b5..24922bc8c79 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -211,7 +211,9 @@ The shared builders live in `src/combos/resolve.ts`, `src/server/admission-model does not resend a response whose bounded body prefix proves context overflow. It reads at most 16,384 bytes from a clone within 250ms or until cancellation, leaving the returned body intact. Event streams and bodies without an observed verdict retain the normal retry; a verdict observed before a slow or -endless body stalls still stops retry. A throttle worded only as "too many tokens" is not a verdict. +endless body stalls still stops retry. Hypothetical, negated, question-form and configuration-failure +mentions are checked against the unwrapped provider message and do not suppress retry; a throttle +worded only as "too many tokens" is not a verdict. Body inspection time does not count toward the upstream slow-attempt budget, and empty chunks cannot starve the deadline or cancellation. The retry and cancellation contracts are exercised in `tests/lib/upstream-retry.test.ts`. diff --git a/tests/lib/upstream-retry.test.ts b/tests/lib/upstream-retry.test.ts index d43d8b14d0a..ef66e797fcd 100644 --- a/tests/lib/upstream-retry.test.ts +++ b/tests/lib/upstream-retry.test.ts @@ -405,6 +405,44 @@ describe("retryBackoffDelayMs", () => { expect(mock.calls).toHaveLength(2); }); + test.each([ + "Your input does not exceed the context window; the service is unavailable", + "Your request doesn't exceed the token limit; retry later", + "Your request exceeds neither the context window nor the token limit; the service is temporarily unavailable", + "if the input exceeds the context window, shorten it; the server is unavailable", + "Failed to load the maximum context length configuration", + "Does the input exceed the context window? The server is unavailable", + ])("a 5xx without an actual overflow verdict remains retryable: %s", async message => { + silenceWarn(); + const mock = mockDoFetch([Response.json({ error: { message } }, { status: 503 }), new Response("ok")]); + expect((await fetchWithTransientRetry(mock.doFetch)).status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + + test("a JSON-escaped negation does not stop a transient retry", async () => { + silenceWarn(); + const body = '{"error":{"message":"Your input does not exceed the context \\u0077indow; retry later"}}'; + const mock = mockDoFetch([new Response(body, { status: 503 }), new Response("ok")]); + expect((await fetchWithTransientRetry(mock.doFetch)).status).toBe(200); + expect(mock.calls).toHaveLength(2); + }); + + test("a definite overflow still stops retries when a later sentence says not to retry", async () => { + silenceWarn(); + const first = new Response("Your input exceeds the context window. Do not retry.", { status: 503 }); + const mock = mockDoFetch([first, new Response("duplicate")]); + expect(await fetchWithTransientRetry(mock.doFetch)).toBe(first); + expect(mock.calls).toHaveLength(1); + }); + + test("a definite overflow stays terminal when its remedy mentions configuration", async () => { + silenceWarn(); + const first = new Response("Your input exceeds the context window; adjust your configuration.", { status: 503 }); + const mock = mockDoFetch([first, new Response("duplicate")]); + expect(await fetchWithTransientRetry(mock.doFetch)).toBe(first); + expect(mock.calls).toHaveLength(1); + }); + test("an instruction inside the wait deadline is still honoured before retrying (#4546)", async () => { silenceWarn(); const limited = new Response("overloaded", { From 13f7daa8b29394ce52efd144bf4170983ca6c4d7 Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 6 Nov 2026 12:47:49 +0900 Subject: [PATCH 3/4] fix(claude): preserve overflow verdicts across sentences --- src/lib/context-overflow-verdict.ts | 7 +++---- src/lib/upstream-retry.ts | 16 +++++++++++++--- src/server/claude-messages.ts | 7 ++++--- structure/data-planes/error-detail.md | 7 +++++-- .../messages-error-normalization.test.ts | 1 + tests/lib/upstream-retry.test.ts | 14 ++++++++++++++ 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/lib/context-overflow-verdict.ts b/src/lib/context-overflow-verdict.ts index 15897bdb030..3e65dace746 100644 --- a/src/lib/context-overflow-verdict.ts +++ b/src/lib/context-overflow-verdict.ts @@ -63,7 +63,6 @@ function isParameterValidationRecord(record: Record): boolean { } function isDefiniteContextOverflowMessage(text: string): boolean { - if (PARAMETER_VALIDATION_MARKERS.some(marker => marker.test(text))) return false; const normalized = text.toLowerCase(); return normalized === "context_length_exceeded" || DEFINITE_CONTEXT_OVERFLOW_PHRASES.some(phrase => normalized.includes(phrase)) @@ -83,10 +82,10 @@ function isDefiniteContextOverflowMessage(text: string): boolean { * arbitrary field that happens to sit in the first 500 bytes authorize a hop, so it fails closed. * * Only the exact proxy wrapper is unwrapped, within a fixed envelope budget and 16,384 characters. - * An optional caller filter can narrow the verdict on the unwrapped leaf, never broaden it. + * An optional caller matcher specializes the verdict on the unwrapped, validated leaf. * Exported for the sidecar loops, which classify their own provider failures with the same rule. */ -export function isDefiniteContextOverflow(status: number, message: string, acceptMessage?: (text: string) => boolean): boolean { +export function isDefiniteContextOverflow(status: number, message: string, matchMessage = isDefiniteContextOverflowMessage): boolean { if (!CONTEXT_VERDICT_STATUSES.has(status) && status < 500) return false; if (message.length > 16_384) return false; let text = message.trim(); @@ -94,7 +93,7 @@ export function isDefiniteContextOverflow(status: number, message: string, accep for (let unwrapped = 0; unwrapped <= MAX_CONTEXT_OVERFLOW_ENVELOPES; unwrapped += 1) { const providerPrefix = /^Provider error \d{3}:\s*/.exec(text); if (providerPrefix) text = text.slice(providerPrefix[0].length).trim(); - if (!text.startsWith("{")) return (acceptMessage?.(text) ?? true) && isDefiniteContextOverflowMessage(text); + if (!text.startsWith("{")) return !PARAMETER_VALIDATION_MARKERS.some(marker => marker.test(text)) && matchMessage(text); if (unwrapped === MAX_CONTEXT_OVERFLOW_ENVELOPES) return false; let payload: unknown; try { payload = JSON.parse(text); } catch { return false; } diff --git a/src/lib/upstream-retry.ts b/src/lib/upstream-retry.ts index 55e6357d268..0c478c73532 100644 --- a/src/lib/upstream-retry.ts +++ b/src/lib/upstream-retry.ts @@ -728,8 +728,19 @@ const CONTEXT_OVERFLOW_PEEK_BYTES = 16_384; const CONTEXT_OVERFLOW_PEEK_MS = 250; // A hypothetical, negated, or configuration-failure mention does not justify suppressing a retry. // Inspect the unwrapped provider message, not its JSON spelling or unrelated envelope fields. -const NON_VERDICT_CONTEXT = /(?:\b(?:not|never|no|neither|nor|if|unless|whether)\b|n['\u2019]t\b)[^.!?\n]{0,100}\b(?:context(?:\s+(?:window|length))?|tokens?|(?:input|prompt)\s+is\s+too\s+long)\b|\b(?:context\s+(?:window|length)|token\s+limit)\b[^.!?\n]{0,40}\?/i; +const NON_VERDICT_CONTEXT = /(?:\b(?:not|never|no|neither|nor|if|unless|whether)\b|n['\u2019]t\b)[^.!?\n]{0,100}\b(?:context(?:\s+(?:window|length))?|tokens?|(?:input|prompt)\s+is\s+too\s+long)\b|\b(?:context\s+(?:window|length)|token\s+limit|(?:input|prompt)\s+is\s+too\s+long)\b[^.!?\n]{0,40}\?/i; const CONTEXT_CONFIG_FAILURE = /\b(?:fail(?:ed|ure)?|unable|cannot|error)\b[^.!?\n]{0,100}\b(?:configuration|settings|documentation)\b|\b(?:configuration|settings|documentation)\b[^.!?\n]{0,100}\b(?:fail(?:ed|ure)?|unavailable|missing|error)\b/i; +const CONTEXT_THROUGHPUT = /\bper (?:second|minute|hour|day)\b|\brate limit|\bquota\b|\btpm\b/i; + +function retryContextOverflowMessage(message: string): boolean { + if (CONTEXT_THROUGHPUT.test(message)) return false; + return message.split(/(?<=[.!?])\s+|[\r\n]+/).some(sentence => { + if (NON_VERDICT_CONTEXT.test(sentence) || CONTEXT_CONFIG_FAILURE.test(sentence)) return false; + const withoutGenericThrottle = sentence.replace(/too many tokens/gi, ""); + return isDefiniteContextOverflow(503, withoutGenericThrottle) + || /\bprompt is too long\b/i.test(sentence); + }); +} /** Up to `maxBytes` of a clone of the body, or what arrived before the deadline. */ async function peekBodyText(res: Response, maxBytes: number, deadlineMs: number, signal?: AbortSignal): Promise { @@ -781,8 +792,7 @@ async function isContextOverflowResponse(res: Response, signal?: AbortSignal): P if ((res.headers.get("content-type") ?? "").includes("text/event-stream")) return false; try { const text = await peekBodyText(res, CONTEXT_OVERFLOW_PEEK_BYTES, CONTEXT_OVERFLOW_PEEK_MS, signal); - return text.length > 0 && isDefiniteContextOverflow(res.status, text.replace(/too many tokens/gi, ""), - message => !NON_VERDICT_CONTEXT.test(message) && !CONTEXT_CONFIG_FAILURE.test(message)); + return text.length > 0 && isDefiniteContextOverflow(res.status, text, retryContextOverflowMessage); } catch { return false; } diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index 710f642df6c..98cf7ffa6fd 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -1339,9 +1339,10 @@ async function handleClaudeMessagesWithBudget( && isContextOverflowError(errorObject, message)); } catch { // A plain-text gateway body carries the same verdict with no envelope to read a code from. - const plain = text.trim().slice(0, 400); - contextError = Boolean(plain) && ![401, 402, 403, 429].includes(response.status) - && isContextOverflowError({}, plain); + const inspected = text.trim().slice(0, 16_384); + const plain = inspected.slice(0, 400); + contextError = Boolean(inspected) && ![401, 402, 403, 429].includes(response.status) + && isContextOverflowError({}, inspected); if (text) message = contextError ? plain : `upstream error (${response.status}): ${text.slice(0, 400)}`; } } catch { /* keep fallback message */ } diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index 24922bc8c79..8d8700f9c36 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -212,8 +212,9 @@ does not resend a response whose bounded body prefix proves context overflow. It bytes from a clone within 250ms or until cancellation, leaving the returned body intact. Event streams and bodies without an observed verdict retain the normal retry; a verdict observed before a slow or endless body stalls still stops retry. Hypothetical, negated, question-form and configuration-failure -mentions are checked against the unwrapped provider message and do not suppress retry; a throttle -worded only as "too many tokens" is not a verdict. +mentions are excluded per sentence in the unwrapped provider message, so they cannot hide a definite +overflow in another sentence. A throttle worded only as "too many tokens" is not a verdict, while an +explicit "prompt is too long" remains a verdict when no throughput signal is present. Body inspection time does not count toward the upstream slow-attempt budget, and empty chunks cannot starve the deadline or cancellation. The retry and cancellation contracts are exercised in `tests/lib/upstream-retry.test.ts`. @@ -382,6 +383,8 @@ to the generic codes; its remaining source and the #8128 suite stay byte-identic `tests/claude-integration/messages-overflow-baseline-contract.test.ts`. The executable overflow golden compares actual client recovery-phrase recognition, HTTP status and the original internal decisions independently on every fixture/path row. The direct Messages writer also applies that shared verdict to the emitted code: a definite generic-coded overflow carries `context_length_exceeded`, the fixed compaction sentence and HTTP 400 on collection, even with `retryable: true`; original error evidence and proxy/transport metadata remain separate. `tests/helpers/messages-overflow-assertions.ts` records the affected fixture ids only on direct-stream/direct-fold as explicit expected changes to the immutable golden. All other paths keep their baseline comparisons; the fixture data is not regenerated. +The translated non-2xx plain-text path classifies up to 16,384 characters before selecting the bounded +400-character display text, so an overflow visible to the retry peek is not lost to display truncation. The direct encoder retains original adapter fields by terminal identity independently of shared classification. Buffered passthrough and count_tokens preserve the upstream status and closed type/code. The type chosen diff --git a/tests/claude-integration/messages-error-normalization.test.ts b/tests/claude-integration/messages-error-normalization.test.ts index 3c833e2e379..5ca5c189f17 100644 --- a/tests/claude-integration/messages-error-normalization.test.ts +++ b/tests/claude-integration/messages-error-normalization.test.ts @@ -114,6 +114,7 @@ test("a direct reader exception quoting overflow wording does not gain an overfl for (const [status, message, expectedStatus, expectedCode] of [ [500, "Your input exceeds the context window of this model", 400, "context_length_exceeded"], + [500, "x".repeat(450) + " Your input exceeds the context window of this model", 400, "context_length_exceeded"], [402, "Your input exceeds the context window of this model", 402, undefined], [429, "Your input exceeds the context window of this model", 429, undefined], ] as const) { diff --git a/tests/lib/upstream-retry.test.ts b/tests/lib/upstream-retry.test.ts index ef66e797fcd..3f1fb30d26b 100644 --- a/tests/lib/upstream-retry.test.ts +++ b/tests/lib/upstream-retry.test.ts @@ -412,6 +412,8 @@ describe("retryBackoffDelayMs", () => { "if the input exceeds the context window, shorten it; the server is unavailable", "Failed to load the maximum context length configuration", "Does the input exceed the context window? The server is unavailable", + "Invalid tool name: 'prompt is too long'", + "Input is too long for username", ])("a 5xx without an actual overflow verdict remains retryable: %s", async message => { silenceWarn(); const mock = mockDoFetch([Response.json({ error: { message } }, { status: 503 }), new Response("ok")]); @@ -443,6 +445,18 @@ describe("retryBackoffDelayMs", () => { expect(mock.calls).toHaveLength(1); }); + test.each([ + "If input exceeds the context window, shorten it. This input exceeds the context window.", + "Failed to load context window configuration. Your input exceeds the context window.", + "Too many tokens: prompt is too long", + ])("a separate or explicit overflow verdict stops a resend: %s", async message => { + silenceWarn(); + const first = new Response(message, { status: 503 }); + const mock = mockDoFetch([first, new Response("duplicate")]); + expect(await fetchWithTransientRetry(mock.doFetch)).toBe(first); + expect(mock.calls).toHaveLength(1); + }); + test("an instruction inside the wait deadline is still honoured before retrying (#4546)", async () => { silenceWarn(); const limited = new Response("overloaded", { From b2e20818fa0d6725f002fa094ad4ea60582fe4bf Mon Sep 17 00:00:00 2001 From: JUN Date: Fri, 6 Nov 2026 12:53:29 +0900 Subject: [PATCH 4/4] fix(claude): preserve verdicts across separate sentences --- src/claude/outbound.ts | 6 ++++-- structure/data-planes/error-detail.md | 3 ++- .../messages-error-normalization.test.ts | 3 +++ .../messages-overflow-baseline-contract.test.ts | 7 +++++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/claude/outbound.ts b/src/claude/outbound.ts index 1a82c6320e5..e40dad22bf8 100644 --- a/src/claude/outbound.ts +++ b/src/claude/outbound.ts @@ -162,8 +162,10 @@ function assertsOverflow(message: string): boolean { if (tooLong && !NEGATION_RE.test(tooLong[1] ?? "") && !CONDITIONAL_LEAD_RE.test(text.slice(0, tooLong.index))) return true; const counts = STATED_COUNTS_RE.exec(text); if (counts && Number(counts[2]) > Number(counts[1])) return true; - const exceeds = EXCEEDS_TEXT_RE.exec(text); - return exceeds !== null && !NEGATION_RE.test(exceeds[0]); + for (const exceeds of text.matchAll(new RegExp(EXCEEDS_TEXT_RE, "gi"))) { + if (!NEGATION_RE.test(exceeds[0]) && !CONDITIONAL_LEAD_RE.test(text.slice(0, exceeds.index))) return true; + } + return false; } /** diff --git a/structure/data-planes/error-detail.md b/structure/data-planes/error-detail.md index 8d8700f9c36..b26d82005c6 100644 --- a/structure/data-planes/error-detail.md +++ b/structure/data-planes/error-detail.md @@ -379,7 +379,8 @@ before selecting the fixed incomplete or compaction sentence. Internal overflow status stay separate from this client-message check: a coded raw-collector throughput refusal still answers 400 without selecting the compaction sentence. The shared `isContextOverflowError` adds `upstream_server_error` -to the generic codes; its remaining source and the #8128 suite stay byte-identical, enforced by +to the generic codes and ignores conditional context-window matches while still checking later matches; +its remaining source and the #8128 suite stay byte-identical, enforced by `tests/claude-integration/messages-overflow-baseline-contract.test.ts`. The executable overflow golden compares actual client recovery-phrase recognition, HTTP status and the original internal decisions independently on every fixture/path row. The direct Messages writer also applies that shared verdict to the emitted code: a definite generic-coded overflow carries `context_length_exceeded`, the fixed compaction sentence and HTTP 400 on collection, even with `retryable: true`; original error evidence and proxy/transport metadata remain separate. `tests/helpers/messages-overflow-assertions.ts` records the affected fixture ids only on direct-stream/direct-fold as explicit expected changes to the immutable golden. All other paths keep their baseline comparisons; the fixture data is not regenerated. diff --git a/tests/claude-integration/messages-error-normalization.test.ts b/tests/claude-integration/messages-error-normalization.test.ts index 5ca5c189f17..acd04a95660 100644 --- a/tests/claude-integration/messages-error-normalization.test.ts +++ b/tests/claude-integration/messages-error-normalization.test.ts @@ -115,6 +115,8 @@ test("a direct reader exception quoting overflow wording does not gain an overfl for (const [status, message, expectedStatus, expectedCode] of [ [500, "Your input exceeds the context window of this model", 400, "context_length_exceeded"], [500, "x".repeat(450) + " Your input exceeds the context window of this model", 400, "context_length_exceeded"], + [500, "If input exceeds the context window, shorten it; the server is unavailable", 529, undefined], + [500, "If input exceeds the context window, shorten it. This input exceeds the context window.", 400, "context_length_exceeded"], [402, "Your input exceeds the context window of this model", 402, undefined], [429, "Your input exceeds the context window of this model", 429, undefined], ] as const) { @@ -139,6 +141,7 @@ for (const [status, message, expectedStatus, expectedCode] of [ expect(error.message).toBe("prompt is too long: The request exceeds the model's context window."); } else { expect(error.code).not.toBe("context_length_exceeded"); + if (status === 500) expect(sends).toBe(3); } } finally { globalThis.fetch = originalFetch; releaseSpend(); } }); diff --git a/tests/claude-integration/messages-overflow-baseline-contract.test.ts b/tests/claude-integration/messages-overflow-baseline-contract.test.ts index 5bcc9c74dd0..64f653c6050 100644 --- a/tests/claude-integration/messages-overflow-baseline-contract.test.ts +++ b/tests/claude-integration/messages-overflow-baseline-contract.test.ts @@ -4,15 +4,18 @@ import { readFileSync } from "node:fs"; import { repoPath } from "../helpers/repo-root"; const digest = (text: string | Uint8Array) => createHash("sha256").update(text).digest("hex"); -test("#8128 shared overflow predicate changes only by adding the generic 5xx code", () => { +test("#8128 shared overflow predicate changes only in the documented 5xx and conditional cases", () => { const source = readFileSync(repoPath("src/claude/outbound.ts"), "utf8"); const start = source.indexOf("const GENERIC_FAILURE_CODES"); const end = source.indexOf("export function anthropicErrorBody", start); expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); const predicate = source.slice(start, end); const addition = '"upstream_server_error", '; + const currentExceeds = ' for (const exceeds of text.matchAll(new RegExp(EXCEEDS_TEXT_RE, "gi"))) {\n if (!NEGATION_RE.test(exceeds[0]) && !CONDITIONAL_LEAD_RE.test(text.slice(0, exceeds.index))) return true;\n }\n return false;'; + const baselineExceeds = ' const exceeds = EXCEEDS_TEXT_RE.exec(text);\n return exceeds !== null && !NEGATION_RE.test(exceeds[0]);'; expect(predicate.split(addition)).toHaveLength(2); - expect(digest(predicate.replace(addition, ""))).toBe("d1d3fd370bcc625c9638f63ff57498456e66b58cf1e258be028bc4d3318493fa"); + expect(predicate.split(currentExceeds)).toHaveLength(2); + expect(digest(predicate.replace(addition, "").replace(currentExceeds, baselineExceeds))).toBe("d1d3fd370bcc625c9638f63ff57498456e66b58cf1e258be028bc4d3318493fa"); }); test("#8128 behavior suite retains origin/dev source", () => { const source = readFileSync(repoPath("tests/claude-integration/claude-uncoded-context-overflow.test.ts"));