diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 7d69cc7b5e3..4651ef4799f 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -685,8 +685,16 @@ the combo returns the last real upstream failure without sending to that target. When `cooldownMs` is unset, a hopped target uses an upstream fallback: 5 seconds for request-rate 429s with upstream code `1302` or `1305`, 10 minutes for a spent account usage window, and 60 seconds otherwise. A usage window is recognised by upstream code (`usage_limit_exceeded`, -`usage_limit_reached`, `1308`) or by the prose `usage limit reached`, independent of HTTP status — -the ChatGPT Codex backend reports a spent window as a 502 rather than the documented 429. Credential and +`usage_limit_reached`, `1308`) or by quota exhaustion phrasing (such as `usage limit reached`, +`token-plan ... quota has been exhausted`, or `weekly usage quota exceeded`) +outside 400 and 422 responses, independent of HTTP status — the ChatGPT Codex backend reports a spent +window as a 502 rather than the documented 429. A 400 or 422 response needs a structured code: a +validation error that merely quotes quota wording (e.g. `quota exhausted`) does not hop. Codes +inferred from that prose by the proxy's formatter do not count as upstream structured evidence. +Devin's explicit weekly/monthly/daily or usage-window exhaustion becomes 429 and uses the +10-minute target hold; per-minute/per-hour and MCP configuration `failed_precondition` trailers stay +400 and stop. Raw Devin trailer text is redacted. A spent +window cools only that target, so other accounts of the same provider stay eligible. Credential and billing failures that already black out the whole provider (`invalid_api_key`, `insufficient_quota`, `payment_required`, and the other provider-scoped codes) take the same 10-minute hold. When it is set, `cooldownMs` applies whenever no usable upstream `Retry-After` or Codex reset signal exists, including those diff --git a/scripts/test-layout/layout-combos.json b/scripts/test-layout/layout-combos.json index 6a4e942a0c9..1426929996f 100644 --- a/scripts/test-layout/layout-combos.json +++ b/scripts/test-layout/layout-combos.json @@ -1,5 +1,6 @@ { "combo-cooldown-reprobe.test.ts": "codex-integration", "server-combo-stream-hosted-tool-contract.test.ts": "server", - "server-combo-stream-send-contract.test.ts": "server" + "server-combo-stream-send-contract.test.ts": "server", + "server-combo-quota-code-provenance.test.ts": "server" } diff --git a/src/adapters/devin/cloud-direct/chat.ts b/src/adapters/devin/cloud-direct/chat.ts index f24877f9a57..435bfca98b0 100644 --- a/src/adapters/devin/cloud-direct/chat.ts +++ b/src/adapters/devin/cloud-direct/chat.ts @@ -1172,6 +1172,18 @@ function safeConnectCode(value: unknown): string | undefined { */ const TRAILER_QUOTA_RE = /\b(?:limit will reset|rate limit|quota exceeded|out of credits)\b/i; +/** + * Spent-window wording. A `failed_precondition` trailer is a 429 only with this evidence, and only + * this evidence earns the synthesized "usage limit reached" label: an ordinary rate limit or a + * per-minute "quota exceeded" stays out of both (#7625). + */ +const TRAILER_EXHAUSTED_RE = /\b(?:usage limit (?:has been )?reached|out of credits|(?:weekly|monthly|daily|usage(?:\s+window)?)\s+(?:usage\s+)?quota (?:has been|is) exhausted|usage window (?:has been|is) exhausted)\b/i; +const TRAILER_TRANSIENT_LIMIT_RE = /\b(?:per[-\s](?:minute|hour)|rate)\b/i; + +function isSpentTrailerWindow(message: string): boolean { + return TRAILER_EXHAUSTED_RE.test(message) && !TRAILER_TRANSIENT_LIMIT_RE.test(message); +} + /** * Connect error code to HTTP status. * @@ -1182,7 +1194,8 @@ const TRAILER_QUOTA_RE = /\b(?:limit will reset|rate limit|quota exceeded|out of * acts on. */ export function connectTrailerHttpStatus(code: string | undefined, message: string): number | undefined { - if (code === 'permission_denied' && TRAILER_QUOTA_RE.test(message)) return 429; + if (code === 'permission_denied' && (TRAILER_QUOTA_RE.test(message) || isSpentTrailerWindow(message))) return 429; + if (code === 'failed_precondition' && isSpentTrailerWindow(message)) return 429; switch (code) { case 'unauthenticated': return 401; case 'permission_denied': return 403; @@ -1635,8 +1648,11 @@ export async function* streamChatEvents(req: CloudChatRequest): AsyncGenerator(); +const STRUCTURED_QUOTA_CODES = new Set(["usage_limit_exceeded", "usage_limit_reached", "1308", "insufficient_quota"]); + +export function upstreamCodeForComboFailure(response: Response, bodyCode: string | undefined): string | undefined { + return bodyCode === "insufficient_quota" && formattedQuotaSources.has(response) + ? formattedQuotaSources.get(response) ?? undefined + : bodyCode; +} + export function formatErrorResponse( status: number, type: string, message: string, - options?: { code?: string | null; retryAfter?: string | null; classified?: boolean }, + options?: { code?: string | null; retryAfter?: string | null; classified?: boolean; upstreamCode?: string | null }, ): Response { + const validationStatus = status === 400 || status === 422; const error = classifyError(status, type, message); // A combo child holding an adapterFailureFromEvent verdict has already classified this // failure. Re-deriving it from the status folded insufficient_quota into rate_limit_exceeded @@ -54,6 +66,11 @@ export function formatErrorResponse( error.code = CYBER_POLICY_ERROR_CODE; error.type = cyberPolicyErrorType(type); } + // Only the adapter event's original code can retain this hop identity. Its classified + // code may have come from reflected prose, and a generic 422 type can erase a real one. + if (validationStatus && options?.upstreamCode + && STRUCTURED_QUOTA_CODES.has(options.upstreamCode.trim().toLowerCase().replaceAll("-", "_")) + && !PINNED_VERDICT_CODES.has(error.code ?? "")) error.code = options.upstreamCode; // The undeclared-tool refusal is deterministic, not a transient upstream failure: its code // survives so the Anthropic translator can classify it instead of folding it into a // retryable overload (and so clients see the SSE path's rejection identity). @@ -94,6 +111,10 @@ export function formatErrorResponse( status: finalStatus, headers, }); + if (validationStatus && error.code === "insufficient_quota") { + formattedQuotaSources.set(response, options?.upstreamCode !== undefined + ? options.upstreamCode : options?.classified ? null : options?.code ?? null); + } if (replayBlocked) markResponseNonReplayable(response); // Re-wrapping is where the refusal loses its provenance: combo failure consumption parses // the JSON and builds a new Response, and the code alone does not tell a later quota diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 3dda95285c5..fd2957eb9aa 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -292,7 +292,7 @@ export function coolComboTarget( // A spent account window or an unpaid/rejected credential does not turn over in a minute, // so the 60s default would re-offer a target that cannot succeed. Only the duration // changes; the scope and hop decisions are untouched. - ?? (isAccountWindowExhausted(options?.message ?? "", options?.code) + ?? (isAccountWindowExhausted(options?.message ?? "", options?.code, options?.status) || PROVIDER_SCOPED_FAILURE_CODES.has(normalizedFailureCode(options?.code)) ? MAX_COOLDOWN_MS : isTransientRequestRateLimit({ @@ -395,11 +395,37 @@ const ACCOUNT_EXHAUSTION_CODES = new Set(["usage_limit_exceeded", "usage_limit_r // Token-plan windows (Alibaba's DeepSeek/Qwen plans) report "Your token-plan 1-week quota has been // exhausted" (#5494). The match is anchored to that phrasing: a looser "quota ... exhausted" would // also catch per-minute limits, and this arm outranks the transient rate-limit duration. -const ACCOUNT_EXHAUSTION_TEXT = /usage limit (?:has been )?reached|token-plan\s+\S+\s+quota has been exhausted/; +const ACCOUNT_EXHAUSTION_TEXT = /usage limit (?:has been )?reached|token-plan\s+\S+\s+quota has been exhausted|(?:account|weekly|monthly|daily)\s*(?:usage\s+)?quota (?:(?:has been |is )?exhausted|exceeded)/i; -function isAccountWindowExhausted(message: string, code?: string | null): boolean { - return ACCOUNT_EXHAUSTION_CODES.has(normalizedFailureCode(code)) - || ACCOUNT_EXHAUSTION_TEXT.test(message.toLowerCase()); +/** + * Determines whether an error signals that an account's quota or usage window has been exhausted. + * + * Explicit error codes (`usage_limit_exceeded`, `insufficient_quota`, etc.) are always authoritative. + * For HTTP 400 (Bad Request), prose alone is not authoritative because client validation errors + * may echo prompt contents or parameter values containing quota-related words. + * + * @param message The error message text. + * @param code The normalized or raw error code. + * @param status The HTTP status code, if known. + * @returns True if the failure represents an account window exhaustion. + */ +function isAccountWindowExhausted( + message: string, + code?: string | null, + status?: number, +): boolean { + const normalized = normalizedFailureCode(code); + if (ACCOUNT_EXHAUSTION_CODES.has(normalized) || normalized === "insufficient_quota") { + return true; + } + // On HTTP 400 (Bad Request / parameter validation), prose alone is not authoritative evidence + // of account quota exhaustion because validation errors may echo user prompts or parameter values + // quoting "quota exhausted". An authoritative account exhaustion verdict on 400 requires a + // structured exhaustion code. + if (status === 400 || status === 422) { + return false; + } + return ACCOUNT_EXHAUSTION_TEXT.test(message.toLowerCase()); } function isProviderScopedQuotaCap( @@ -419,6 +445,12 @@ function isProviderScopedQuotaCap( || (text.includes("free tier") && text.includes("single request")); } +/** A structured account-window exhaustion code; prose never qualifies (a request can echo it). */ +function hasAccountExhaustionCode(code?: string | null): boolean { + const normalized = normalizedFailureCode(code); + return ACCOUNT_EXHAUSTION_CODES.has(normalized) || normalized === "insufficient_quota"; +} + /** * A free-tier cap the upstream evaluates PER REQUEST rather than per account window. These * needles used to reach only `isProviderScopedQuotaCap`, so a single oversized free-tier prompt @@ -918,9 +950,16 @@ export function comboFailureDecision( // adapter grants it (the words alone prove nothing for another provider); it must precede the // generic invalid_request_error stop. if (options?.failedAdapter === "kiro" && isKiroAccountProfileRefusal(status, message)) return "hop"; + // A structured account-window exhaustion code is provider-state evidence even inside a + // 400 / invalid_request_error envelope, so it must precede that stop. Exhaustion prose alone + // never reaches this arm: a validation error can quote quota wording back (#7625). + if (hasAccountExhaustionCode(options?.code)) return "hop"; if (["origin_rejected", "context_length_exceeded", "invalid_request_error"].includes(error.code ?? "")) { return "stop"; } + // Quota prose inside a request-validation status (400/422) is not authoritative without a + // structured code: the body can quote prompt or parameter text back (#7625). + if (error.code === "insufficient_quota" && (status === 400 || status === 422)) return "stop"; // 402 (payment required) and 425 (too early) are provider-state signals, not verdicts about // the request: another combo target can still serve it. if ([401, 402, 403, 404, 408, 425, 429].includes(status) || status >= 500) return "hop"; diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 1bdcae0477f..443ede90fa1 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -500,8 +500,10 @@ export function classifyError(status: number, type: string, message: string): Oc text.includes("insufficient_quota") || text.includes("exceeded your current quota") || text.includes("quota exhausted") || + text.includes("quota has been exhausted") || text.includes("account quota exceeded") || text.includes("monthly quota exceeded") || + text.includes("weekly quota exceeded") || text.includes("daily quota exceeded") ) { return { message, type: "insufficient_quota", code: "insufficient_quota" }; diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index f17a52868f6..211c1c18a5f 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -18,6 +18,7 @@ import { import { normalizeUpstreamErrorText } from "./core-errors"; import { resolveClientRetryAfter } from "../../lib/retry-after"; import { formatErrorResponse } from "../../bridge"; +import { upstreamCodeForComboFailure } from "../../bridge/errors"; import { OLLAMA_LATE_ATTRIBUTION_LIMIT_MESSAGE } from "../../adapters/ollama-native"; import { isNonReplayableResponse, markResponseNonReplayable } from "../../lib/upstream-retry"; import { MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope"; @@ -76,7 +77,8 @@ export async function consumeComboFailure( const normalized = normalizeUpstreamErrorText(body.text, fallback); if (utf8Trusted || isCyberPolicyCode(normalized.code) || isCyberPolicyMessage(normalized.safeText)) { classificationText = normalized.safeText; - upstreamCode = codexAccountModelRefusalHardStopCode(response.status, body.text) ?? normalized.code; + upstreamCode = codexAccountModelRefusalHardStopCode(response.status, body.text) + ?? upstreamCodeForComboFailure(response, normalized.code); // Only the adapter's own normalized refusal may claim the plan-refusal hop; another // upstream echoing the code in any spelling keeps the meaning it had before, which was none. if (namesModelNotInPlanCode(upstreamCode) && !isModelNotInPlanRefusal(response, upstreamCode)) upstreamCode = undefined; diff --git a/src/server/responses/run-turn-execution.ts b/src/server/responses/run-turn-execution.ts index c5555b917d1..e2f6277004f 100644 --- a/src/server/responses/run-turn-execution.ts +++ b/src/server/responses/run-turn-execution.ts @@ -841,7 +841,9 @@ export async function executeResponsesRunTurn( const storageRefusal = preflight.error?.code === SPEND_LEDGER_STORAGE_UNAVAILABLE_CODE; const failure = formatErrorResponse(storageRefusal ? 503 : classified.httpStatus, storageRefusal ? SPEND_LEDGER_STORAGE_UNAVAILABLE_CODE : classified.error.type, - redactSecretString(classified.error.message), { code: classified.error.code, classified: true }); + redactSecretString(classified.error.message), { + code: classified.error.code, classified: true, upstreamCode: preflight.error?.code ?? null, + }); // A replay-unsafe heartbeat means the adapter already ran a local side effect, so the // combo must not send this turn to another target: the failure stays with this child. if (preflight.replayUnsafe) markResponseNonReplayable(failure); @@ -1047,7 +1049,10 @@ export async function executeResponsesRunTurn( const storageRefusal = firstMeaningful?.type === "error" && firstMeaningful.code === SPEND_LEDGER_STORAGE_UNAVAILABLE_CODE; const failure = formatErrorResponse(storageRefusal ? 503 : classified.httpStatus, storageRefusal ? SPEND_LEDGER_STORAGE_UNAVAILABLE_CODE : classified.error.type, - redactSecretString(classified.error.message), { code: classified.error.code, classified: true }); + redactSecretString(classified.error.message), { + code: classified.error.code, classified: true, + upstreamCode: classifiedError?.code ?? (firstMeaningful?.type === "error" ? firstMeaningful.code : null) ?? null, + }); if (replayUnsafe) markResponseNonReplayable(failure); return failure; } diff --git a/structure/transports/responses-combo-boundaries.md b/structure/transports/responses-combo-boundaries.md index 6511eb4e0c3..0068233f701 100644 --- a/structure/transports/responses-combo-boundaries.md +++ b/structure/transports/responses-combo-boundaries.md @@ -122,3 +122,21 @@ deprecated, sunset, decommissioned, or no longer available). An unrelated applic not retried. > Decision record: [ADR-0071](../decisions/ADR-0071-combo-streaming-commit-boundary.md) + +## Upstream account quota exhaustion failover + +A structured account-window exhaustion code (`usage_limit_exceeded`, `usage_limit_reached`, `1308`, `insufficient_quota`) makes `comboFailureDecision` hop even inside a 400 / `invalid_request_error` envelope, ahead of that terminal stop. Exhaustion prose alone never reaches that arm, and quota prose inside a 400 or 422 response stops instead of hopping: a request-validation body can quote prompt or parameter text back. `ACCOUNT_EXHAUSTION_TEXT` (`usage limit reached`, `token-plan ... quota has been exhausted`, and account/weekly/monthly/daily quota exhausted or exceeded wording) only lengthens a target's cooldown to `MAX_COOLDOWN_MS` on other statuses. + +`src/server/responses/run-turn-execution.ts` passes the original adapter event code separately +from its classified verdict to `src/bridge/errors.ts`. When formatting a 400/422 produces +`insufficient_quota`, a response-identity WeakMap retains that original code (or its absence). +`src/server/responses/core-combo-failure.ts` reads that provenance before treating the serialized +code as upstream evidence. A prose-derived quota code cannot authorize a hop or provider scope +on 400/422; genuine upstream exhaustion codes remain authoritative on both statuses. Rebuilt +failure responses retain the same distinction. Raw upstream envelopes carry their own code; +the formatter's client-facing classification and cyber-policy precedence remain intact. + +Account exhaustion stays target-scoped in `comboFailureCooldownScope`: it names one account, and a provider row can carry several pooled accounts whose healthy siblings must stay eligible on the next request. Provider scope remains reserved for provider-wide evidence (rejected or unpaid credentials, the existing `GoUsageLimitError` / monthly-cap and billing arms). + +The Devin direct chat adapter (`src/adapters/devin/cloud-direct/chat.ts`) maps a Connect `failed_precondition` trailer to 429 only when it carries explicit spent-window wording (`TRAILER_EXHAUSTED_RE`: usage limit reached, out of credits, a weekly/monthly/daily/usage quota or usage window that has been or is exhausted). Per-minute, per-hour and rate wording vetoes that evidence. Only a 429 spent window earns the constant `usage limit reached` label, which survives formatting as target-local exhaustion with the 10-minute fallback instead of becoming provider-scoped `insufficient_quota`. Raw trailer text stays inside the parser. Unqualified exhaustion, transient quota preconditions and request-specific preconditions such as MCP configuration errors stay 400 and stop. +Coverage: `tests/routing/router-combo-failover-classification.test.ts`, `tests/providers/devin-hardening.test.ts`, `tests/server/error-fidelity.test.ts`, `tests/codex-integration/combo-codex-exhaustion-cooldown.test.ts`, `tests/server/server-combo-quota-code-provenance.test.ts`, and `tests/responses/responses-combo-run-turn-classified-failure.test.ts`. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 80f24460672..703f5637d88 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -11,6 +11,7 @@ "fix-eventstream-cancel-wait.test.ts": "responses", "native-source-priority.test.ts": "codex-integration", "server-native-catalog-priority.test.ts": "server", + "server-combo-quota-code-provenance.test.ts": "server", "claude-intercept-acl-barrier.test.ts": "claude-integration", "claude-config-root-writer-barrier.test.ts": "claude-integration", "claude-config-root-late-admission.test.ts": "claude-integration", diff --git a/tests/providers/devin-hardening.test.ts b/tests/providers/devin-hardening.test.ts index e7d5e575a00..6b300ef1749 100644 --- a/tests/providers/devin-hardening.test.ts +++ b/tests/providers/devin-hardening.test.ts @@ -414,6 +414,16 @@ describe("connect trailer to HTTP status", () => { expect(connectTrailerHttpStatus("permission_denied", "an internal error occurred")).toBe(403); }); + test("quota exhaustion delivered as failed_precondition is a 429, not a 400", () => { + expect(connectTrailerHttpStatus("failed_precondition", "Your weekly usage quota has been exhausted. Visit https://app.devin.ai/settings/usage to purchase on-demand usage or turn on auto-reload.")).toBe(429); + expect(connectTrailerHttpStatus("failed_precondition", "usage limit reached")).toBe(429); + expect(connectTrailerHttpStatus("failed_precondition", "Unable to process request due to an MCP configuration issue.")).toBe(400); + // A precondition that only mentions a rate limit or a per-minute quota is not a spent window. + expect(connectTrailerHttpStatus("failed_precondition", "rate limit configuration is invalid")).toBe(400); + expect(connectTrailerHttpStatus("failed_precondition", "per-minute quota exceeded for this tool")).toBe(400); + expect(connectTrailerHttpStatus("permission_denied", "Your weekly usage quota has been exhausted.")).toBe(429); + }); + test("the remaining Connect codes map to the status core acts on", () => { expect(connectTrailerHttpStatus("unauthenticated", "")).toBe(401); expect(connectTrailerHttpStatus("resource_exhausted", "")).toBe(429); @@ -684,6 +694,51 @@ describe("devin cloud trailer errors", () => { } }); + test("synthesizes account-window copy when trailer matches quota pattern", async () => { + const credential = "devin-session-token$header.payload.signature"; + const trailer = Buffer.from(JSON.stringify({ + error: { + code: "failed_precondition", + message: `Your weekly usage quota has been exhausted. Visit https://app.devin.ai/settings/usage to purchase on-demand usage. ${credential}`, + }, + })); + const envelope = Buffer.alloc(5 + trailer.length); + envelope[0] = 0x02; + envelope.writeUInt32BE(trailer.length, 1); + trailer.copy(envelope, 5); + + const originalFetch = globalThis.fetch; + clearCachedCatalog(); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + return calls === 1 + ? new Response("catalog unavailable", { status: 503 }) + : new Response(envelope, { status: 200 }); + }) as typeof fetch; + try { + let caught: unknown; + try { + for await (const _event of streamChatEvents({ + apiKey: credential, + modelUid: "swe-2-high", + messages: [{ role: "user", content: "hi" }], + })) { /* no data frames */ } + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect((caught as any).status).toBe(429); + expect((caught as any).code).toBe("failed_precondition"); + expect((caught as Error).message).toContain("usage limit reached"); + expect((caught as Error).message).not.toContain(credential); + expect((caught as Error).message).not.toContain("Your weekly usage quota"); + } finally { + globalThis.fetch = originalFetch; + clearCachedCatalog(); + } + }); + test("carries our own retry-seconds wording without the raw trailer text", async () => { const credential = "devin-session-token$header.payload.signature"; const trailer = Buffer.from(JSON.stringify({ diff --git a/tests/responses/responses-combo-run-turn-classified-failure.test.ts b/tests/responses/responses-combo-run-turn-classified-failure.test.ts index 934124521f3..0fd42249fe4 100644 --- a/tests/responses/responses-combo-run-turn-classified-failure.test.ts +++ b/tests/responses/responses-combo-run-turn-classified-failure.test.ts @@ -116,6 +116,36 @@ afterEach(() => { }); for (const stream of [false, true]) { + for (const status of [400, 422]) { + for (const code of [undefined, "invalid_argument"]) { + test(`reflected quota prose stays terminal through runTurn (stream=${stream}, status=${status}, code=${code})`, async () => { + childFailure = { + type: "error", status, code, errorType: "invalid_request_error", + message: "Invalid parameter: value quotes 'weekly usage quota exceeded; quota exhausted'", + }; + const cfg = config([{ provider: "cursor", model: MODEL }, { provider: "fallback", model: MODEL }]); + const response = await respond(cfg, stream); + expect(response.status).toBe(status); + expect(runTurnCalls).toBe(1); + expect(fallbackCalls).toBe(0); + // A validation error did not cool the healthy provider for a later valid request. + childFailure = undefined; + const next = await respond(cfg, stream); + expect(next.status).toBe(200); + expect(next.text).toContain("cursor answered"); + expect(runTurnCalls).toBe(2); + expect(fallbackCalls).toBe(0); + }); + } + test(`a structured spent window survives runTurn formatting (stream=${stream}, status=${status})`, async () => { + childFailure = { type: "error", status, code: "usage_limit_reached", message: "Account window spent" }; + const response = await respond(config([{ provider: "cursor", model: MODEL }, { provider: "fallback", model: MODEL }]), stream); + expect(response.status).toBe(200); + expect(runTurnCalls).toBe(1); + expect(fallbackCalls).toBe(1); + expect(response.text).toContain("fallback answered"); + }); + } test(`a child that cannot take image input hands the turn to the next target (stream=${stream})`, async () => { childFailure = { type: "error", status: 400, errorType: "invalid_request_error", code: "unsupported_input_modality", diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index b9b05088687..8e60fc9cef8 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -104,10 +104,74 @@ describe("combo failure cooldown scope", () => { expect(comboFailureCooldownScope(429, "monthly usage limit reached")).toBe("provider"); }); + test("one account's spent window cools only that target, so sibling accounts stay eligible", () => { + // An account-window exhaustion names one account, not the provider: a provider row can carry + // several pooled accounts, and a provider-wide hold would hide the healthy ones (#7426, #7625). + for (const message of [ + "quota exhausted", + "weekly usage quota exceeded", + "weekly usage quota is exhausted", + "Your weekly usage quota has been exhausted.", + "Devin cloud error failed_precondition: Cognition chat failed (failed_precondition: quota exhausted) (cloud trace ID: 0123abcd)", + ]) expect(comboFailureCooldownScope(429, message)).toBe("target"); + expect(comboFailureCooldownScope(429, "quota exhausted for this minute")).toBe("target"); + expect(comboFailureCooldownScope(429, "usage limit reached", { code: "usage_limit_reached" })).toBe("target"); + }); + test("an ordinary target failure still cools only that target", () => { expect(comboFailureCooldownScope(500, "internal server error")).toBe("target"); expect(comboFailureCooldownScope(429, "rate limit reached for requests")).toBe("target"); }); + + test("transient per-minute rate limits and quotas cool only that target and preserve short fallback duration", () => { + // A per-minute quota or rate limit is transient and must not be promoted to provider-wide + // exhaustion (which uses the 10-minute MAX_COOLDOWN_MS and provider scope). + expect(comboFailureCooldownScope(429, "per-minute quota exceeded")).toBe("target"); + expect(comboFailureCooldownScope(429, "Rate limit reached: per-minute quota exceeded")).toBe("target"); + expect(comboFailureCooldownScope(429, "requests per minute quota exceeded")).toBe("target"); + + const now = 1_000_000; + const target = { provider: "a", model: "m1" }; + + // Per-minute quota gets standard 60s cooldown, not the 10-minute (600s) account exhaustion cap. + coolComboTarget("free", target, { + now, + status: 429, + message: "per-minute quota exceeded", + }); + expect(isComboTargetInCooldown("free", target, now + 59_000)).toBe(true); + expect(isComboTargetInCooldown("free", target, now + 61_000)).toBe(false); + + // In contrast, weekly usage quota exhaustion gets the 10-minute cap. + coolComboTarget("free", target, { + now, + status: 429, + message: "Your weekly usage quota has been exhausted.", + }); + expect(isComboTargetInCooldown("free", target, now + 599_000)).toBe(true); + expect(isComboTargetInCooldown("free", target, now + 601_000)).toBe(false); + }); + + test("a 400 validation error quoting quota text does not trigger account failover or provider cooldown", () => { + const errorQuotingQuota = "Invalid parameter 'prompt': text cannot contain 'quota exhausted'"; + expect(comboFailureDecision(400, errorQuotingQuota, { code: "invalid_request_error" })).toBe("stop"); + expect(comboFailureCooldownScope(400, errorQuotingQuota, { code: "invalid_request_error" })).not.toBe("provider"); + + const errorQuotingExhaustion = "Validation error: argument quotes 'Your weekly usage quota has been exhausted'"; + expect(comboFailureDecision(400, errorQuotingExhaustion)).toBe("stop"); + expect(comboFailureCooldownScope(400, errorQuotingExhaustion)).not.toBe("provider"); + + // The same reflected wording on a 422, with or without the upstream's own invalid_request_error code. + const reflected422 = "Invalid parameter: value quotes 'weekly usage quota exceeded'"; + expect(comboFailureDecision(422, reflected422, { code: "invalid_request_error" })).toBe("stop"); + expect(comboFailureDecision(422, reflected422)).toBe("stop"); + }); + + test("a structured exhaustion code hops even inside a 400 envelope", () => { + for (const code of ["usage_limit_exceeded", "usage_limit_reached", "1308", "insufficient_quota"]) { + expect(comboFailureDecision(400, "Bad request", { code })).toBe("hop"); + } + }); }); describe("combo failure hop/stop verdicts", () => { @@ -117,6 +181,21 @@ describe("combo failure hop/stop verdicts", () => { } }); + test("devin quota exhaustion arrives as a 429 and hops, while request-specific precondition stops", () => { + const exhausted = "Devin cloud error failed_precondition: Cognition chat failed (failed_precondition: quota exhausted) (cloud trace ID: 0123abcd)"; + expect(comboFailureDecision(429, exhausted, { code: "failed_precondition", failedAdapter: "devin" })).toBe("hop"); + expect(comboFailureCooldownScope(429, exhausted, { code: "failed_precondition" })).toBe("target"); + + // Request-specific precondition without quota evidence stays terminal and does not cool provider + expect(comboFailureDecision(400, "Devin cloud error failed_precondition: Unable to process request due to an MCP configuration issue", { + code: "failed_precondition", + failedAdapter: "devin", + })).toBe("stop"); + expect(comboFailureCooldownScope(400, "Devin cloud error failed_precondition: Unable to process request due to an MCP configuration issue", { + code: "failed_precondition", + })).not.toBe("provider"); + }); + test("a Codex model the ChatGPT plan cannot use hops instead of ending the chain", () => { const body = JSON.stringify({ detail: "The 'gpt-6-astra' model is not supported when using Codex with a ChatGPT account." }); expect(comboFailureDecision(400, body)).toBe("hop"); diff --git a/tests/server/error-fidelity.test.ts b/tests/server/error-fidelity.test.ts index 151637088a8..569ccc88f8c 100644 --- a/tests/server/error-fidelity.test.ts +++ b/tests/server/error-fidelity.test.ts @@ -47,6 +47,14 @@ describe("error fidelity", () => { type: "insufficient_quota", code: "insufficient_quota", }); + expect(classifyError(400, "upstream_error", "Your weekly usage quota has been exhausted")).toMatchObject({ + type: "insufficient_quota", + code: "insufficient_quota", + }); + expect(classifyError(400, "invalid_request_error", "Invalid configuration: weekly usage quota must be greater than zero")).toMatchObject({ + type: "invalid_request_error", + code: "invalid_request_error", + }); expect(classifyError(403, "origin_rejected", "WebSocket upgrade blocked: non-local Origin")).toMatchObject({ type: "invalid_request_error", code: "origin_rejected", diff --git a/tests/server/server-combo-quota-code-provenance.test.ts b/tests/server/server-combo-quota-code-provenance.test.ts new file mode 100644 index 00000000000..c2507b46a39 --- /dev/null +++ b/tests/server/server-combo-quota-code-provenance.test.ts @@ -0,0 +1,156 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { formatErrorResponse } from "../../src/bridge"; +import { adapterFailureFromEvent } from "../../src/bridge/internal"; +import { devinErrorClassification } from "../../src/adapters/devin"; +import { clearCachedCatalog } from "../../src/adapters/devin/cloud-direct/catalog"; +import { CloudChatError, streamChatEvents } from "../../src/adapters/devin/cloud-direct/chat"; +import { + clearComboTargetCooldowns, comboFailureCooldownScope, comboFailureDecision, + coolComboTarget, isComboTargetInCooldown, +} from "../../src/combos/failover"; +import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; +import type { AdapterEvent } from "../../src/types"; + +const combo = "quota-code-provenance"; +const target = { provider: "devin", model: "swe-2-high" }; +const sibling = { provider: "devin", model: "swe-2-low" }; +const now = 1_000_000; +const credential = "devin-session-token$fixture.payload.signature"; +afterEach(() => clearComboTargetCooldowns(combo)); + +async function consumeAdapterError(event: Extract) { + const { httpStatus, error } = adapterFailureFromEvent(event); + // Same child formatter boundary as run-turn-execution, including the original event code. + return consumeComboFailure(formatErrorResponse(httpStatus, error.type, error.message, { + code: error.code, classified: true, upstreamCode: event.code ?? null, + })); +} + +async function devinTrailer(code: string, message: string): Promise { + const trailer = Buffer.from(JSON.stringify({ error: { code, message: `${message} ${credential}` } })); + const envelope = Buffer.alloc(5 + trailer.length); + envelope[0] = 0x02; + envelope.writeUInt32BE(trailer.length, 1); + trailer.copy(envelope, 5); + const originalFetch = globalThis.fetch; + clearCachedCatalog(); + let calls = 0; + globalThis.fetch = (async () => ++calls === 1 + ? new Response("catalog unavailable", { status: 503 }) + : new Response(envelope, { status: 200 })) as typeof fetch; + try { + try { + for await (const _ of streamChatEvents({ + apiKey: credential, modelUid: target.model, messages: [{ role: "user", content: "hi" }], + })) { /* trailer-only stream */ } + } catch (error) { + expect(error).toBeInstanceOf(CloudChatError); + return error as CloudChatError; + } + throw new Error("expected a trailer refusal"); + } finally { + globalThis.fetch = originalFetch; + clearCachedCatalog(); + } +} + +async function consumeDevin(error: CloudChatError) { + return consumeAdapterError({ + type: "error", code: error.code, ...devinErrorClassification(error), + message: `Devin cloud error${error.code ? ` ${error.code}` : ""}: ${error.message}`, + }); +} + +describe("quota evidence through child formatting and combo consumption", () => { + test("R1: a real Devin weekly window hops, cools only the target for ten minutes, and redacts", async () => { + const error = await devinTrailer("failed_precondition", "Your weekly usage quota has been exhausted."); + const failure = await consumeDevin(error); + expect(failure.response.status).toBe(429); + expect(comboFailureDecision(429, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); + expect(comboFailureCooldownScope(429, failure.classificationText, { code: failure.upstreamCode })).toBe("target"); + coolComboTarget(combo, target, { now, status: 429, code: failure.upstreamCode, message: failure.classificationText }); + expect(isComboTargetInCooldown(combo, target, now + 599_999)).toBe(true); + expect(isComboTargetInCooldown(combo, target, now + 600_000)).toBe(false); + expect(isComboTargetInCooldown(combo, sibling, now)).toBe(false); + const body = await failure.response.text(); + expect(body).not.toContain(credential); + expect(body).not.toContain("Your weekly usage quota"); + }); + + for (const message of [ + "Your monthly quota is exhausted", "Your daily usage quota has been exhausted", + "Your usage window is exhausted", "Usage limit reached", "Out of credits", + ]) { + test(`explicit Devin spent-window evidence retains target scope: ${message}`, async () => { + const failure = await consumeDevin(await devinTrailer("failed_precondition", message)); + expect(failure.response.status).toBe(429); + expect(comboFailureDecision(429, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); + expect(comboFailureCooldownScope(429, failure.classificationText, { code: failure.upstreamCode })).toBe("target"); + coolComboTarget(combo, target, { now, status: 429, code: failure.upstreamCode, message: failure.classificationText }); + expect(isComboTargetInCooldown(combo, target, now + 599_999)).toBe(true); + expect(isComboTargetInCooldown(combo, target, now + 600_000)).toBe(false); + }); + } + + for (const status of [400, 422]) { + for (const phrase of ["weekly usage quota exceeded", "quota exhausted", "quota has been exhausted", "insufficient_quota"]) { + test(`R2: HTTP ${status} reflected '${phrase}' never supplies structured exhaustion`, async () => { + const failure = await consumeAdapterError({ + type: "error", status, message: `Invalid parameter: value quotes '${phrase}'`, + }); + expect(failure.response.status).toBe(status); + expect(failure.upstreamCode).not.toBe("insufficient_quota"); + expect(comboFailureDecision(status, failure.classificationText, { code: failure.upstreamCode })).toBe("stop"); + expect(comboFailureCooldownScope(status, failure.classificationText, { code: failure.upstreamCode })).not.toBe("provider"); + coolComboTarget(combo, target, { now, status, code: failure.upstreamCode, message: failure.classificationText }); + expect(isComboTargetInCooldown(combo, target, now + 60_000)).toBe(false); + }); + } + test(`R2: HTTP ${status} unclassified formatter prose cannot mint upstream quota evidence`, async () => { + const failure = await consumeComboFailure(formatErrorResponse(status, "upstream_error", "Invalid value: quota exhausted")); + expect(comboFailureDecision(status, failure.classificationText, { code: failure.upstreamCode })).toBe("stop"); + expect(comboFailureCooldownScope(status, failure.classificationText, { code: failure.upstreamCode })).not.toBe("provider"); + }); + for (const code of ["usage_limit_exceeded", "usage_limit_reached", "1308", "insufficient_quota"]) { + test(`HTTP ${status} genuine upstream ${code} still authorizes a hop`, async () => { + const failure = await consumeAdapterError({ type: "error", status, code, message: "Account window spent" }); + expect(failure.upstreamCode).toBe(code); + expect(comboFailureDecision(status, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); + }); + } + } + + test("R2: a real Devin invalid_argument reflecting weekly quota text stops", async () => { + const error = await devinTrailer("invalid_argument", "Invalid value: Your weekly usage quota has been exhausted"); + const failure = await consumeDevin(error); + expect(failure.response.status).toBe(400); + expect(comboFailureDecision(400, failure.classificationText, { code: failure.upstreamCode })).toBe("stop"); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode })).not.toBe("provider"); + }); + + for (const message of [ + "per-minute quota is exhausted for this tool", + "per-minute quota has been exhausted for this tool", + "per-hour quota is exhausted for this tool", + "quota has been exhausted for this tool", + "per-minute usage quota is exhausted for this tool", + "usage limit reached for requests per minute", + "weekly quota is exhausted by a rate limit", + ]) { + test(`R3: transient or unqualified precondition stays 400: ${message}`, async () => { + const error = await devinTrailer("failed_precondition", message); + const failure = await consumeDevin(error); + expect(error.status).toBe(400); + expect(error.message).not.toContain("usage limit reached"); + expect(failure.response.status).toBe(400); + expect(comboFailureDecision(400, failure.classificationText, { code: failure.upstreamCode })).toBe("stop"); + }); + } + + test("MCP configuration preconditions remain redacted 400 stops", async () => { + const failure = await consumeDevin(await devinTrailer("failed_precondition", "Unable to process request due to an MCP configuration issue")); + expect(failure.response.status).toBe(400); + expect(comboFailureDecision(400, failure.classificationText, { code: failure.upstreamCode })).toBe("stop"); + expect(await failure.response.text()).not.toContain(credential); + }); +});