From 6f724a178545431c8b0c6264ab786a74f2d83823 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:08:31 +0600 Subject: [PATCH 01/29] fix(combo): fallback on provider moderation refusal --- src/combos/failover.ts | 85 ++++++++++++++++++ ...uter-combo-failover-classification.test.ts | 88 ++++++++++++++++++- 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 7dbafe75fab..d20e098b649 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -644,6 +644,89 @@ function isResponseFormatCapabilityRefusal( return false; } +function namesModerationRefusal(text: string): boolean { + const lower = text.toLowerCase(); + return ( + lower.includes("refused by moderation") + || lower.includes("blocked by moderation") + || lower.includes("flagged by moderation") + || lower.includes("content management policy") + || (lower.includes("moderation") && lower.includes("categories:")) + ); +} + +/** + * A provider or gateway moderation filter refusal is target-local in a heterogeneous combo: + * this model or provider's guardrail refused the prompt, but another target may serve it. + * + * It must precede the generic invalid_request_error stop. + */ +export function isProviderModerationRefusal( + status: number, + message: string, + code?: string | null, +): boolean { + if (status !== 400 && status !== 422) return false; + if (!message || message.length > 32_768) return false; + const normalizedCode = normalizedFailureCode(code); + if ( + isCyberPolicyCode(code) + || isNonReplayableUpstreamCode(code) + || normalizedCode === "origin_rejected" + ) { + return false; + } + if (normalizedCode === "moderation_refusal" || normalizedCode === "content_filter") { + return true; + } + + let text = message.trim(); + for (let depth = 0; depth < 3; depth += 1) { + if (text.startsWith("Provider error 400: ") || text.startsWith("Provider error 422: ")) { + text = text.slice(text.indexOf(": ") + 2).trim(); + } + if (text.startsWith("data:") && !text.includes("\n")) { + text = text.slice("data:".length).trim(); + } + let payload: unknown; + try { + payload = JSON.parse(text); + } catch { + break; + } + if (!payload || typeof payload !== "object" || Array.isArray(payload)) break; + const rec = payload as Record; + const error = rec.error ?? rec; + if (!error || typeof error !== "object" || Array.isArray(error)) break; + const e = error as Record; + + if (typeof e.message === "string" && (e.message.startsWith("Provider error 400: ") || e.message.startsWith("Provider error 422: "))) { + text = e.message; + continue; + } + + const candidateTexts: string[] = []; + if (typeof e.message === "string") candidateTexts.push(e.message); + if (typeof e.error === "string") candidateTexts.push(e.error); + if (typeof rec.message === "string") candidateTexts.push(rec.message); + if (e.param && typeof e.param === "object" && !Array.isArray(e.param)) { + const p = e.param as Record; + if (typeof p.message === "string") candidateTexts.push(p.message); + if (typeof p.error === "string") candidateTexts.push(p.error); + } + + const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); + if (errCode === "moderation_refusal" || errCode === "content_filter") return true; + + for (const msg of candidateTexts) { + if (namesModerationRefusal(msg)) return true; + } + return false; + } + + return namesModerationRefusal(text); +} + export function comboFailureCooldownScope( status: number, message: string, @@ -665,6 +748,7 @@ export function comboFailureCooldownScope( // A capability gap says the target is healthy and the request did not fit it, which is the // same reason every other entry here refuses to cool a target. || isResponseFormatCapabilityRefusal(status, message, options?.code) + || isProviderModerationRefusal(status, message, options?.code) ) return "none"; if (isProviderScopedQuotaCap(status, message, options?.code)) return "provider"; // A rejected or unpaid credential is provider-wide evidence: every target that routes @@ -879,6 +963,7 @@ export function comboFailureDecision( // reaches that list and returns terminal before anything can ask whether the next target // could have served the request (#4903). if (isResponseFormatCapabilityRefusal(status, message, options?.code)) return "hop"; + if (isProviderModerationRefusal(status, message, options?.code)) return "hop"; if (isAnthropicBillingFailure(status, message, options?.code)) return "hop"; // Kiro's body validator rejects the request for the identity it was routed under. The account // layer rotates logins first; this is the no-alternate case, so the next combo target is the diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 2a3d7ca7f6a..b05e202eadd 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -8,7 +8,7 @@ import { pickComboTarget, targetKey, } from "../../src/combos"; -import { codexAccountModelRefusalHardStopCode, comboFailureCooldownScope, comboFailureDecision } from "../../src/combos/failover"; +import { codexAccountModelRefusalHardStopCode, comboFailureCooldownScope, comboFailureDecision, isProviderModerationRefusal } from "../../src/combos/failover"; import { adapterFailureFromMessage, inferHttpStatusFromAdapterMessage } from "../../src/lib/errors"; import type { OcxConfig } from "../../src/types"; @@ -715,3 +715,89 @@ describe("HTTP hard-code evidence excludes diagnostic types", () => { } }); }); + +describe("provider moderation refusal", () => { + const moderationPayload = { + error: { + message: "The input was refused by moderation. Categories: harassment.", + type: "AI_APICallError", + param: { + error: "The input was refused by moderation. Categories: harassment.", + statusCode: 400, + name: "AI_APICallError", + message: "The input was refused by moderation. Categories: harassment.", + isRetryable: false, + type: "AI_APICallError", + }, + }, + providerMetadata: { + gateway: { + routing: { + originalModelId: "humansand/ampersand-v0.1", + resolvedProvider: "humansand", + fallbacksAvailable: [], + }, + }, + }, + }; + + test("hops without cooling on observed gateway moderation refusal", () => { + const rawBody = JSON.stringify(moderationPayload); + expect(isProviderModerationRefusal(400, rawBody)).toBe(true); + for (const message of [rawBody, `Provider error 400: ${rawBody}`, `data: ${rawBody}`]) { + expect(comboFailureDecision(400, message)).toBe("hop"); + expect(comboFailureCooldownScope(400, message)).toBe("none"); + } + }); + + test("hops on various provider moderation phrases", () => { + const phrases = [ + "The input was refused by moderation. Categories: harassment.", + "The input was refused by moderation.", + "The prompt was refused by moderation.", + "The response was refused by moderation.", + "Prompt was blocked by moderation.", + "Input was flagged by moderation.", + "Request was filtered due to the prompt triggering content management policy.", + ]; + for (const phrase of phrases) { + const body = JSON.stringify({ error: { message: phrase } }); + expect(isProviderModerationRefusal(400, body)).toBe(true); + expect(comboFailureDecision(400, body)).toBe("hop"); + expect(comboFailureCooldownScope(400, body)).toBe("none"); + } + }); + + test("hops when error code is moderation_refusal or content_filter", () => { + for (const code of ["moderation_refusal", "content_filter"]) { + const body = JSON.stringify({ error: { code, message: "filtered" } }); + expect(isProviderModerationRefusal(400, body, code)).toBe(true); + expect(comboFailureDecision(400, body, { code })).toBe("hop"); + expect(comboFailureCooldownScope(400, body, { code })).toBe("none"); + } + }); + + test("hard stops like cyber_policy outrank moderation wording", () => { + const body = JSON.stringify({ + error: { + code: "cyber_policy", + message: "The input was refused by moderation.", + }, + }); + expect(isProviderModerationRefusal(400, body, "cyber_policy")).toBe(false); + expect(comboFailureDecision(400, body, { code: "cyber_policy" })).toBe("stop"); + }); + + test("generic 400 errors without moderation phrasing stay terminal", () => { + for (const message of [ + "Invalid JSON in request body", + "Missing required parameter 'messages'", + "Unknown model 'some-model'", + ]) { + const body = JSON.stringify({ error: { message } }); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + } + }); +}); + From 92b729918b15d9c8bf3d4994f3309caa93b08f26 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:25:45 +0600 Subject: [PATCH 02/29] fix(combo): harden moderation classification against reflections and wrapper hard codes --- src/combos/failover.ts | 55 ++++++++++--- ...uter-combo-failover-classification.test.ts | 82 +++++++++++++++++-- 2 files changed, 120 insertions(+), 17 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index d20e098b649..f488c543203 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -644,7 +644,22 @@ function isResponseFormatCapabilityRefusal( return false; } +function isReflectedValidationText(text: string): boolean { + const lower = text.toLowerCase(); + return ( + lower.includes("invalid parameter") + || lower.includes("invalid value") + || lower.includes("validation error") + || lower.includes("cannot contain") + || lower.includes("quotes '") + || lower.includes("quotes \"") + || /quotes\s+['"]/i.test(text) + || /is not supported for (?:field|parameter)\b/i.test(text) + ); +} + function namesModerationRefusal(text: string): boolean { + if (isReflectedValidationText(text)) return false; const lower = text.toLowerCase(); return ( lower.includes("refused by moderation") @@ -667,11 +682,10 @@ export function isProviderModerationRefusal( code?: string | null, ): boolean { if (status !== 400 && status !== 422) return false; - if (!message || message.length > 32_768) return false; const normalizedCode = normalizedFailureCode(code); if ( - isCyberPolicyCode(code) - || isNonReplayableUpstreamCode(code) + isCyberPolicyCode(normalizedCode) + || isNonReplayableUpstreamCode(normalizedCode) || normalizedCode === "origin_rejected" ) { return false; @@ -679,8 +693,10 @@ export function isProviderModerationRefusal( if (normalizedCode === "moderation_refusal" || normalizedCode === "content_filter") { return true; } + if (!message || message.length > 32_768) return false; let text = message.trim(); + let wasParsed = false; for (let depth = 0; depth < 3; depth += 1) { if (text.startsWith("Provider error 400: ") || text.startsWith("Provider error 422: ")) { text = text.slice(text.indexOf(": ") + 2).trim(); @@ -691,20 +707,38 @@ export function isProviderModerationRefusal( let payload: unknown; try { payload = JSON.parse(text); + wasParsed = true; } catch { + if (text.startsWith("{") || text.startsWith("[") || wasParsed) { + return false; + } break; } - if (!payload || typeof payload !== "object" || Array.isArray(payload)) break; + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false; const rec = payload as Record; const error = rec.error ?? rec; - if (!error || typeof error !== "object" || Array.isArray(error)) break; + if (!error || typeof error !== "object" || Array.isArray(error)) return false; const e = error as Record; + const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : typeof rec.code === "string" ? rec.code : undefined); + if ( + isCyberPolicyCode(errCode) + || isNonReplayableUpstreamCode(errCode) + || errCode === "origin_rejected" + ) { + return false; + } + if (errCode === "moderation_refusal" || errCode === "content_filter") return true; + if (typeof e.message === "string" && (e.message.startsWith("Provider error 400: ") || e.message.startsWith("Provider error 422: "))) { text = e.message; continue; } + if (typeof e.param === "string" && e.param && !["prompt", "input", "messages"].includes(e.param.trim().toLowerCase())) { + return false; + } + const candidateTexts: string[] = []; if (typeof e.message === "string") candidateTexts.push(e.message); if (typeof e.error === "string") candidateTexts.push(e.error); @@ -715,15 +749,13 @@ export function isProviderModerationRefusal( if (typeof p.error === "string") candidateTexts.push(p.error); } - const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); - if (errCode === "moderation_refusal" || errCode === "content_filter") return true; - for (const msg of candidateTexts) { if (namesModerationRefusal(msg)) return true; } return false; } + if (wasParsed || text.startsWith("{") || text.startsWith("[")) return false; return namesModerationRefusal(text); } @@ -883,14 +915,15 @@ export function comboFailureDecision( // Structured form of the same hard refusal. The prose test above misses it when the origin // reports the code out of band, and every hop rule below -- including the context-overflow // one -- must stay subordinate to it. - if (normalizedFailureCode(options?.code) === "origin_rejected") return "stop"; + const normalizedCode = normalizedFailureCode(options?.code); + if (normalizedCode === "origin_rejected") return "stop"; // The origin may already be executing this turn (the Codex WebSocket relay sent the create // frame and never saw a response event). Hopping would send the same request to a second // target while the first may still be generating; the honest status goes to the client. - if (isNonReplayableUpstreamCode(options?.code)) return "stop"; + if (isNonReplayableUpstreamCode(normalizedCode)) return "stop"; // Cyber policy is a hard non-retryable refusal — honor structured code even when // classificationText was truncated before the JSON code field. - if (isCyberPolicyCode(options?.code)) return "stop"; + if (isCyberPolicyCode(normalizedCode)) return "stop"; // HTTP consumers retain hard codes before truncation; SSE metadata owns its selected carrier. if (options?.codexModelRefusal === undefined && codexAccountModelRefusalHardStopCode(status, message)) return "stop"; const modelRefusal = status === 400 diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index b05e202eadd..9c7fc978ae5 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -774,18 +774,88 @@ describe("provider moderation refusal", () => { expect(isProviderModerationRefusal(400, body, code)).toBe(true); expect(comboFailureDecision(400, body, { code })).toBe("hop"); expect(comboFailureCooldownScope(400, body, { code })).toBe("none"); + for (const message of [body, `data: ${body}`]) { + expect(isProviderModerationRefusal(400, message)).toBe(true); + expect(comboFailureDecision(400, message)).toBe("hop"); + expect(comboFailureCooldownScope(400, message)).toBe("none"); + } + } + }); + + test("recognizes explicit moderation codes with empty or absent message", () => { + for (const code of ["content_filter", "moderation_refusal"]) { + expect(isProviderModerationRefusal(400, "", code)).toBe(true); + expect(comboFailureDecision(400, "", { code })).toBe("hop"); + expect(comboFailureCooldownScope(400, "", { code })).toBe("none"); + + const body = JSON.stringify({ error: { code } }); + expect(isProviderModerationRefusal(400, body)).toBe(true); + expect(comboFailureDecision(400, body)).toBe("hop"); + } + }); + + test("hard stops like cyber_policy and upstream_no_response outrank moderation wording", () => { + for (const code of ["cyber_policy", "CYBER_POLICY", "upstream_no_response", "UPSTREAM_NO_RESPONSE", "origin_rejected"]) { + const body = JSON.stringify({ + error: { + code, + message: "The input was refused by moderation.", + }, + }); + expect(isProviderModerationRefusal(400, body, code)).toBe(false); + expect(comboFailureDecision(400, body, { code })).toBe("stop"); + + for (const message of [body, `data: ${body}`]) { + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + } } }); - test("hard stops like cyber_policy outrank moderation wording", () => { - const body = JSON.stringify({ + test("truncated 422 with uppercase CYBER_POLICY code stays terminal", () => { + const raw = JSON.stringify({ + error: { + message: "The input was refused by moderation. Categories: harassment.", + code: "CYBER_POLICY", + }, + }); + // Truncated at 500 characters + const truncated = `Provider error 422: ${raw.slice(0, 80)}`; + expect(comboFailureDecision(422, truncated, { code: "CYBER_POLICY" })).toBe("stop"); + expect(isProviderModerationRefusal(422, truncated, "CYBER_POLICY")).toBe(false); + }); + + test("reflected validation text quoting moderation phrases stays terminal", () => { + const reflected = [ + "Invalid parameter 'prompt': text cannot contain 'blocked by moderation'", + "Validation error: argument quotes 'refused by moderation'", + "Invalid parameter: value quotes 'blocked by moderation'", + "The value 'blocked by moderation' is not supported for field 'prompt'", + ]; + for (const message of reflected) { + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + + const body = JSON.stringify({ error: { message } }); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + } + }); + + test("truncated or malformed JSON with moderation wording in unrelated fields fails closed", () => { + const malformed = '{"error":{"message":"Invalid input","note":"blocked by moderation"'; + expect(isProviderModerationRefusal(400, malformed)).toBe(false); + expect(comboFailureDecision(400, malformed)).toBe("stop"); + + const unrelatedField = JSON.stringify({ error: { - code: "cyber_policy", - message: "The input was refused by moderation.", + message: "Invalid JSON input syntax", + param: "notes", + unrelated_note: "blocked by moderation", }, }); - expect(isProviderModerationRefusal(400, body, "cyber_policy")).toBe(false); - expect(comboFailureDecision(400, body, { code: "cyber_policy" })).toBe("stop"); + expect(isProviderModerationRefusal(400, unrelatedField)).toBe(false); + expect(comboFailureDecision(400, unrelatedField)).toBe("stop"); }); test("generic 400 errors without moderation phrasing stay terminal", () => { From 200cdc702e7a733400f536bab0e234995563c709 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:58:37 +0600 Subject: [PATCH 03/29] fix(combo): address CodeRabbit review feedback on moderation fallbacks --- src/combos/failover.ts | 83 ++++++++++++++----- ...uter-combo-failover-classification.test.ts | 76 +++++++++++++++++ 2 files changed, 138 insertions(+), 21 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index f488c543203..4d0c8b53a62 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { classifyError, isCyberPolicyCode } from "../lib/errors"; +import { classifyError, isCyberPolicyCode, isCyberPolicyMessage } from "../lib/errors"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; import { isKiroAccountProfileRefusal } from "../adapters/kiro-refusal"; @@ -644,17 +644,29 @@ function isResponseFormatCapabilityRefusal( return false; } +function isAllowedModerationParam(param: string): boolean { + const p = param.trim().toLowerCase(); + return ( + p === "prompt" + || p === "input" + || p === "messages" + || p.startsWith("input[") + || p.startsWith("input.") + || p.startsWith("messages[") + || p.startsWith("messages.") + ); +} + function isReflectedValidationText(text: string): boolean { const lower = text.toLowerCase(); return ( - lower.includes("invalid parameter") - || lower.includes("invalid value") - || lower.includes("validation error") - || lower.includes("cannot contain") + lower.includes("cannot contain") || lower.includes("quotes '") || lower.includes("quotes \"") || /quotes\s+['"]/i.test(text) + || /text cannot contain/i.test(text) || /is not supported for (?:field|parameter)\b/i.test(text) + || /invalid (?:parameter|value) '.*?'/i.test(text) ); } @@ -690,13 +702,15 @@ export function isProviderModerationRefusal( ) { return false; } - if (normalizedCode === "moderation_refusal" || normalizedCode === "content_filter") { - return true; + if (!message) { + return normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; } - if (!message || message.length > 32_768) return false; + if (message.length > 32_768) return false; let text = message.trim(); let wasParsed = false; + let moderationCodeSeen = normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; + for (let depth = 0; depth < 3; depth += 1) { if (text.startsWith("Provider error 400: ") || text.startsWith("Provider error 422: ")) { text = text.slice(text.indexOf(": ") + 2).trim(); @@ -710,6 +724,22 @@ export function isProviderModerationRefusal( wasParsed = true; } catch { if (text.startsWith("{") || text.startsWith("[") || wasParsed) { + const match = /"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); + if (match) { + try { + const extractedMsg = JSON.parse(`"${match[1]}"`) as string; + if ( + !isCyberPolicyMessage(text) + && !isNonReplayableUpstreamCode(normalizedCode) + && normalizedCode !== "origin_rejected" + && namesModerationRefusal(extractedMsg) + ) { + return true; + } + } catch { + return false; + } + } return false; } break; @@ -720,25 +750,35 @@ export function isProviderModerationRefusal( if (!error || typeof error !== "object" || Array.isArray(error)) return false; const e = error as Record; - const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : typeof rec.code === "string" ? rec.code : undefined); - if ( - isCyberPolicyCode(errCode) - || isNonReplayableUpstreamCode(errCode) - || errCode === "origin_rejected" - ) { - return false; + const rootCode = normalizedFailureCode(typeof rec.code === "string" ? rec.code : undefined); + const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); + for (const c of [rootCode, errCode]) { + if ( + isCyberPolicyCode(c) + || isNonReplayableUpstreamCode(c) + || c === "origin_rejected" + ) { + return false; + } + if (c === "moderation_refusal" || c === "content_filter") { + moderationCodeSeen = true; + } } - if (errCode === "moderation_refusal" || errCode === "content_filter") return true; - if (typeof e.message === "string" && (e.message.startsWith("Provider error 400: ") || e.message.startsWith("Provider error 422: "))) { - text = e.message; - continue; + if (typeof e.message === "string") { + const msgTrimmed = e.message.trim(); + if (msgTrimmed.startsWith("Provider error 400: ") || msgTrimmed.startsWith("Provider error 422: ") || msgTrimmed.startsWith("{")) { + text = msgTrimmed; + continue; + } } - if (typeof e.param === "string" && e.param && !["prompt", "input", "messages"].includes(e.param.trim().toLowerCase())) { + if (typeof e.param === "string" && e.param && !isAllowedModerationParam(e.param)) { return false; } + if (moderationCodeSeen) return true; + const candidateTexts: string[] = []; if (typeof e.message === "string") candidateTexts.push(e.message); if (typeof e.error === "string") candidateTexts.push(e.error); @@ -755,7 +795,8 @@ export function isProviderModerationRefusal( return false; } - if (wasParsed || text.startsWith("{") || text.startsWith("[")) return false; + if (moderationCodeSeen) return true; + if (wasParsed || text.startsWith("{") || text.startsWith("[") || text.startsWith("data:")) return false; return namesModerationRefusal(text); } diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 9c7fc978ae5..f908d60f567 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -858,6 +858,82 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, unrelatedField)).toBe("stop"); }); + test("direct moderation verdict after validation prefix hops", () => { + const directValidation = "Validation error: the prompt was blocked by moderation"; + expect(isProviderModerationRefusal(400, directValidation)).toBe(true); + expect(comboFailureDecision(400, directValidation)).toBe("hop"); + expect(comboFailureCooldownScope(400, directValidation)).toBe("none"); + + const body = JSON.stringify({ error: { message: directValidation } }); + expect(isProviderModerationRefusal(400, body)).toBe(true); + expect(comboFailureDecision(400, body)).toBe("hop"); + }); + + test("accepts indexed content parameter paths in moderation refusals", () => { + for (const param of ["input[0].content", "messages[0].content", "messages[1].content[0].text", "input.prompt"]) { + const body = JSON.stringify({ + error: { + message: "The input was refused by moderation. Categories: harassment.", + param, + }, + }); + expect(isProviderModerationRefusal(400, body)).toBe(true); + expect(comboFailureDecision(400, body)).toBe("hop"); + } + }); + + test("multi-event SSE body with unparsed diagnostic note stays terminal", () => { + const multiEventSse = 'data: {"event":"start"}\ndata: {"note":"blocked by moderation"}\ndata: {"error":"bad_request"}'; + expect(isProviderModerationRefusal(400, multiEventSse)).toBe(false); + expect(comboFailureDecision(400, multiEventSse)).toBe("stop"); + }); + + test("mixed-code hard stops outrank competing wrappers and nested moderation evidence", () => { + // 422 JSON and single-line SSE with a root upstream_no_response code and nested moderation evidence + const rootUpstreamNoResponse = JSON.stringify({ + code: "upstream_no_response", + error: { + message: "The input was refused by moderation. Categories: harassment.", + }, + }); + expect(isProviderModerationRefusal(422, rootUpstreamNoResponse)).toBe(false); + expect(comboFailureDecision(422, rootUpstreamNoResponse)).toBe("stop"); + + const sseRootUpstreamNoResponse = `data: ${rootUpstreamNoResponse}`; + expect(isProviderModerationRefusal(422, sseRootUpstreamNoResponse)).toBe(false); + expect(comboFailureDecision(422, sseRootUpstreamNoResponse)).toBe("stop"); + + // Moderation-coded wrapper around an inner hard code + const moderationWrapperHardCode = JSON.stringify({ + error: { + code: "moderation_refusal", + type: "upstream_error", + message: JSON.stringify({ + error: { + code: "cyber_policy", + message: "Blocked by cyber policy", + }, + }), + }, + }); + expect(isProviderModerationRefusal(400, moderationWrapperHardCode)).toBe(false); + expect(comboFailureDecision(400, moderationWrapperHardCode)).toBe("stop"); + }); + + test("truncated JSON envelope over 500 characters with intact moderation message hops", () => { + const padding = "x".repeat(600); + const raw = JSON.stringify({ + error: { + message: "The input was refused by moderation. Categories: harassment.", + padding, + }, + }); + // Truncate after the message field (at 200 characters) + const truncated = raw.slice(0, 200); + expect(isProviderModerationRefusal(400, truncated)).toBe(true); + expect(comboFailureDecision(400, truncated)).toBe("hop"); + }); + test("generic 400 errors without moderation phrasing stay terminal", () => { for (const message of [ "Invalid JSON in request body", From ea1971055dc9443ad267b1a082861507c2bc8443 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:13:53 +0600 Subject: [PATCH 04/29] fix(combo): address CodeRabbit review feedback on malformed envelopes and wrapper limits --- src/combos/failover.ts | 37 ++++++++---- ...uter-combo-failover-classification.test.ts | 60 ++++++++++++++++++- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 4d0c8b53a62..a828b6f9ef8 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -724,16 +724,29 @@ export function isProviderModerationRefusal( wasParsed = true; } catch { if (text.startsWith("{") || text.startsWith("[") || wasParsed) { - const match = /"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); + if ( + isCyberPolicyMessage(text) + || /["']?(?:upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused|cyber_policy|origin_rejected)["']?/i.test(text) + || isNonReplayableUpstreamCode(normalizedCode) + || normalizedCode === "origin_rejected" + ) { + return false; + } + const paramMatch = /"param"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); + if (paramMatch) { + try { + const extractedParam = JSON.parse(`"${paramMatch[1]}"`) as string; + if (!isAllowedModerationParam(extractedParam)) return false; + } catch { + return false; + } + } + const match = /"(?:error|response)"\s*:\s*\{[^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text) + ?? /^\s*\{\s*"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); if (match) { try { const extractedMsg = JSON.parse(`"${match[1]}"`) as string; - if ( - !isCyberPolicyMessage(text) - && !isNonReplayableUpstreamCode(normalizedCode) - && normalizedCode !== "origin_rejected" - && namesModerationRefusal(extractedMsg) - ) { + if (namesModerationRefusal(extractedMsg)) { return true; } } catch { @@ -765,6 +778,10 @@ export function isProviderModerationRefusal( } } + if (typeof e.param === "string" && e.param && !isAllowedModerationParam(e.param)) { + return false; + } + if (typeof e.message === "string") { const msgTrimmed = e.message.trim(); if (msgTrimmed.startsWith("Provider error 400: ") || msgTrimmed.startsWith("Provider error 422: ") || msgTrimmed.startsWith("{")) { @@ -773,10 +790,6 @@ export function isProviderModerationRefusal( } } - if (typeof e.param === "string" && e.param && !isAllowedModerationParam(e.param)) { - return false; - } - if (moderationCodeSeen) return true; const candidateTexts: string[] = []; @@ -795,8 +808,8 @@ export function isProviderModerationRefusal( return false; } - if (moderationCodeSeen) return true; if (wasParsed || text.startsWith("{") || text.startsWith("[") || text.startsWith("data:")) return false; + if (moderationCodeSeen) return true; return namesModerationRefusal(text); } diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index f908d60f567..b673de5b207 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -9,6 +9,7 @@ import { targetKey, } from "../../src/combos"; import { codexAccountModelRefusalHardStopCode, comboFailureCooldownScope, comboFailureDecision, isProviderModerationRefusal } from "../../src/combos/failover"; +import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; import { adapterFailureFromMessage, inferHttpStatusFromAdapterMessage } from "../../src/lib/errors"; import type { OcxConfig } from "../../src/types"; @@ -920,7 +921,7 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, moderationWrapperHardCode)).toBe("stop"); }); - test("truncated JSON envelope over 500 characters with intact moderation message hops", () => { + test("truncated JSON envelope over 500 characters with intact moderation message hops", async () => { const padding = "x".repeat(600); const raw = JSON.stringify({ error: { @@ -932,6 +933,63 @@ describe("provider moderation refusal", () => { const truncated = raw.slice(0, 200); expect(isProviderModerationRefusal(400, truncated)).toBe(true); expect(comboFailureDecision(400, truncated)).toBe("hop"); + + // Pass through consumeComboFailure where production truncation occurs + const response = new Response(raw, { status: 400 }); + const failure = await consumeComboFailure(response); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("none"); + }); + + test("malformed JSON restricts recovery to error envelope and rejects non-error message fields", () => { + // moderation wording in diagnostic.message rather than error.message + const diagnosticMessage = '{"diagnostic":{"message":"blocked by moderation"},"error":{"message":"invalid param"}}'.slice(0, 75); + expect(isProviderModerationRefusal(400, diagnosticMessage)).toBe(false); + expect(comboFailureDecision(400, diagnosticMessage)).toBe("stop"); + + // malformed 422 with upstream_no_response code and moderation phrasing + const malformedHardStop = '{"code":"upstream_no_response","error":{"message":"The input was refused by moderation"'.slice(0, 70); + expect(isProviderModerationRefusal(422, malformedHardStop)).toBe(false); + expect(comboFailureDecision(422, malformedHardStop)).toBe("stop"); + }); + + test("outer non-moderation parameter check precedes message unwrapping", () => { + const outerBadParam = JSON.stringify({ + error: { + param: "temperature", + message: JSON.stringify({ + error: { + message: "The input was refused by moderation. Categories: harassment.", + }, + }), + }, + }); + expect(isProviderModerationRefusal(400, outerBadParam)).toBe(false); + expect(comboFailureDecision(400, outerBadParam)).toBe("stop"); + }); + + test("wrapper limit exhaustion fails closed on unexamined structured text", () => { + const deepConcealedHardCode = JSON.stringify({ + code: "moderation_refusal", + error: { + message: JSON.stringify({ + error: { + message: JSON.stringify({ + error: { + message: JSON.stringify({ + error: { + code: "upstream_no_response", + message: "timed out", + }, + }), + }, + }), + }, + }), + }, + }); + expect(isProviderModerationRefusal(400, deepConcealedHardCode)).toBe(false); + expect(comboFailureDecision(400, deepConcealedHardCode)).toBe("stop"); }); test("generic 400 errors without moderation phrasing stay terminal", () => { From f4ae8b9943062a4f3284d9d9925e796db9e61c9f Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:35:48 +0600 Subject: [PATCH 05/29] fix(combo): scope malformed parameter checks and allow explicit moderation code on input fields --- src/combos/failover.ts | 35 +++++++++++-------- ...uter-combo-failover-classification.test.ts | 27 ++++++++++++++ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index a828b6f9ef8..598c4ef03df 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -732,20 +732,23 @@ export function isProviderModerationRefusal( ) { return false; } - const paramMatch = /"param"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); - if (paramMatch) { - try { - const extractedParam = JSON.parse(`"${paramMatch[1]}"`) as string; - if (!isAllowedModerationParam(extractedParam)) return false; - } catch { - return false; + // Match error/response envelope containing message and optionally param + const envelopeMatch = /"(?:error|response)"\s*:\s*\{([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text) + ?? /^\s*\{\s*([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text); + if (envelopeMatch) { + const envelopeContent = envelopeMatch[1]; + const rawMessage = envelopeMatch[2]; + const paramMatch = /"param"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(envelopeContent); + if (paramMatch) { + try { + const extractedParam = JSON.parse(`"${paramMatch[1]}"`) as string; + if (!isAllowedModerationParam(extractedParam)) return false; + } catch { + return false; + } } - } - const match = /"(?:error|response)"\s*:\s*\{[^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text) - ?? /^\s*\{\s*"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(text); - if (match) { try { - const extractedMsg = JSON.parse(`"${match[1]}"`) as string; + const extractedMsg = JSON.parse(`"${rawMessage}"`) as string; if (namesModerationRefusal(extractedMsg)) { return true; } @@ -765,6 +768,7 @@ export function isProviderModerationRefusal( const rootCode = normalizedFailureCode(typeof rec.code === "string" ? rec.code : undefined); const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); + let explicitErrorModerationCode = false; for (const c of [rootCode, errCode]) { if ( isCyberPolicyCode(c) @@ -775,11 +779,14 @@ export function isProviderModerationRefusal( } if (c === "moderation_refusal" || c === "content_filter") { moderationCodeSeen = true; + explicitErrorModerationCode = true; } } - if (typeof e.param === "string" && e.param && !isAllowedModerationParam(e.param)) { - return false; + if (typeof e.param === "string" && e.param) { + if (!isAllowedModerationParam(e.param) && !explicitErrorModerationCode) { + return false; + } } if (typeof e.message === "string") { diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index b673de5b207..ac4576d6bb9 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -992,6 +992,33 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, deepConcealedHardCode)).toBe("stop"); }); + test("unrelated param preceding error envelope in truncated response does not block refusal", async () => { + const raw = JSON.stringify({ + request_info: { param: "unrelated_header" }, + error: { + message: "The input was refused by moderation. Categories: harassment.", + padding: "y".repeat(600), + }, + }); + const response = new Response(raw, { status: 400 }); + const failure = await consumeComboFailure(response); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("none"); + }); + + test("explicit content_filter code triggers failover even when param names other input field", () => { + const body = JSON.stringify({ + error: { + code: "content_filter", + message: "Your input violated content safety policy.", + param: "instructions", + }, + }); + expect(isProviderModerationRefusal(400, body)).toBe(true); + expect(comboFailureDecision(400, body)).toBe("hop"); + expect(comboFailureCooldownScope(400, body)).toBe("none"); + }); + test("generic 400 errors without moderation phrasing stay terminal", () => { for (const message of [ "Invalid JSON in request body", From c1916f430e404bb704ed0e4098cb6d6c4fc695a3 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Mon, 9 Nov 2026 16:53:00 +0600 Subject: [PATCH 06/29] fix(combo): prioritize error envelope in truncated responses and restrict param override to error object --- src/combos/failover.ts | 10 +++---- ...uter-combo-failover-classification.test.ts | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 598c4ef03df..6002cfe8e53 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -732,9 +732,10 @@ export function isProviderModerationRefusal( ) { return false; } - // Match error/response envelope containing message and optionally param + // Match authoritative error/response envelope containing message and optionally param. + // A dedicated "error": { ... } or "response": { ... } takes precedence over root message. const envelopeMatch = /"(?:error|response)"\s*:\s*\{([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text) - ?? /^\s*\{\s*([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text); + ?? (!/"(?:error|response)"\s*:\s*\{/.test(text) ? /^\{\s*([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text) : null); if (envelopeMatch) { const envelopeContent = envelopeMatch[1]; const rawMessage = envelopeMatch[2]; @@ -768,7 +769,6 @@ export function isProviderModerationRefusal( const rootCode = normalizedFailureCode(typeof rec.code === "string" ? rec.code : undefined); const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); - let explicitErrorModerationCode = false; for (const c of [rootCode, errCode]) { if ( isCyberPolicyCode(c) @@ -779,12 +779,12 @@ export function isProviderModerationRefusal( } if (c === "moderation_refusal" || c === "content_filter") { moderationCodeSeen = true; - explicitErrorModerationCode = true; } } + const errorObjectModerationCode = errCode === "moderation_refusal" || errCode === "content_filter"; if (typeof e.param === "string" && e.param) { - if (!isAllowedModerationParam(e.param) && !explicitErrorModerationCode) { + if (!isAllowedModerationParam(e.param) && !errorObjectModerationCode) { return false; } } diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index ac4576d6bb9..3d2e6ee59c3 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1019,6 +1019,33 @@ describe("provider moderation refusal", () => { expect(comboFailureCooldownScope(400, body)).toBe("none"); }); + test("root content_filter code does not exempt nested validation error with disallowed param", () => { + const body = JSON.stringify({ + code: "content_filter", + error: { + message: "Invalid temperature: must be between 0 and 2", + param: "temperature", + }, + }); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + }); + + test("truncated body with root moderation message and nested validation error prefers error envelope", () => { + const raw = JSON.stringify({ + message: "The input was refused by moderation. Categories: harassment.", + extra: { info: "nested" }, + error: { + message: "Invalid parameter: prompt is missing", + param: "prompt", + padding: "z".repeat(600), + }, + }); + const truncated = raw.slice(0, 180); + expect(isProviderModerationRefusal(400, truncated)).toBe(false); + expect(comboFailureDecision(400, truncated)).toBe("stop"); + }); + test("generic 400 errors without moderation phrasing stay terminal", () => { for (const message of [ "Invalid JSON in request body", From d57cd6e420b023e9b35885ac8b1f3cd348914c35 Mon Sep 17 00:00:00 2001 From: Omar <13569609+omar391@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:54:54 +0600 Subject: [PATCH 07/29] fix(combo): require unquoted verdict evidence for moderation refusals A 400/422 validation error that quotes a moderation phrase as a parameter value ("Invalid enum value 'blocked by moderation'") was classified as a provider moderation refusal, so the combo replayed the invalid request on another target instead of returning the validation error. namesModerationRefusal now requires each verdict phrase to occur outside quoted spans: a moderation phrase that only appears inside quotes is a reflected parameter value, not a verdict. Apostrophes inside words are treated as literal text so ordinary prose is unaffected. --- src/combos/failover.ts | 64 +++++++++++++++++-- ...uter-combo-failover-classification.test.ts | 24 +++++++ 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 6002cfe8e53..4cbf0427522 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -670,16 +670,66 @@ function isReflectedValidationText(text: string): boolean { ); } +/** + * A moderation phrase echoed inside quotes is a reflected parameter value (`Invalid enum value + * 'blocked by moderation'`), not a verdict. Track quote state so verdict evidence must appear + * in prose; an apostrophe inside a word ("don't") is literal text, not a quote delimiter. + */ +function isInsideQuotedSpan(text: string, end: number): boolean { + let inSingle = false; + let inDouble = false; + for (let i = 0; i < end; i += 1) { + const ch = text[i]; + if (ch === "\\") { + i += 1; + continue; + } + if (inDouble) { + if (ch === "\"") inDouble = false; + continue; + } + if (inSingle) { + if (ch === "'") inSingle = false; + continue; + } + if (ch === "\"") { + inDouble = true; + continue; + } + if ( + ch === "'" + && !(i > 0 && i + 1 < text.length && /[A-Za-z]/.test(text[i - 1]) && /[A-Za-z]/.test(text[i + 1])) + ) { + inSingle = true; + } + } + return inSingle || inDouble; +} + +function hasUnquotedOccurrence(text: string, needle: string): boolean { + let from = 0; + for (;;) { + const idx = text.indexOf(needle, from); + if (idx === -1) return false; + if (!isInsideQuotedSpan(text, idx)) return true; + from = idx + 1; + } +} + +const MODERATION_REFUSAL_VERDICT_PHRASES = [ + "refused by moderation", + "blocked by moderation", + "flagged by moderation", + "content management policy", +]; + function namesModerationRefusal(text: string): boolean { if (isReflectedValidationText(text)) return false; const lower = text.toLowerCase(); - return ( - lower.includes("refused by moderation") - || lower.includes("blocked by moderation") - || lower.includes("flagged by moderation") - || lower.includes("content management policy") - || (lower.includes("moderation") && lower.includes("categories:")) - ); + for (const phrase of MODERATION_REFUSAL_VERDICT_PHRASES) { + if (hasUnquotedOccurrence(lower, phrase)) return true; + } + return hasUnquotedOccurrence(lower, "moderation") && hasUnquotedOccurrence(lower, "categories:"); } /** diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 3d2e6ee59c3..66006fdc16e 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -843,6 +843,30 @@ describe("provider moderation refusal", () => { } }); + test("quoted moderation phrases in validation values stay terminal", () => { + const reflected = [ + "Invalid enum value 'blocked by moderation' for param 'prompt'", + 'The value "blocked by moderation" is invalid', + "param 'mode' does not accept \"flagged by moderation\"", + "Unknown value 'content management policy' supplied for 'reason'", + "Unsupported value 'refused by moderation' at messages[0].content", + ]; + for (const message of reflected) { + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + + const body = JSON.stringify({ error: { message, param: "prompt" } }); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + } + }); + + test("truncated envelope quoting a moderation value stays terminal", () => { + const truncated = '{"error":{"message":"Invalid enum value \\"blocked by moderation\\" for param","param":"prompt"'; + expect(isProviderModerationRefusal(400, truncated)).toBe(false); + expect(comboFailureDecision(400, truncated)).toBe("stop"); + }); + test("truncated or malformed JSON with moderation wording in unrelated fields fails closed", () => { const malformed = '{"error":{"message":"Invalid input","note":"blocked by moderation"'; expect(isProviderModerationRefusal(400, malformed)).toBe(false); From 0a61dac2e09e2e87930f6ec6ea96a3d78424b466 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 01:55:35 +0900 Subject: [PATCH 08/29] fix(combo): never let a moderation verdict outrank a hard refusal Independent review found bodies that are terminal on dev but hopped with this change: invalid_prompt, bio_policy and misalignment codes beside moderation prose, safety-system copy, a terminal root code over a nested content_filter, a nested hard code, and a serialized non-replayable root behind an unstripped gateway prefix. A validation error that merely named moderation categories also hopped and skipped cooldown. isProviderModerationRefusal now returns false for any terminal refusal code, any hard-refusal or non-replayable token anywhere in the body, and any body the hard-refusal parser or the safety-copy matcher already classifies. Only explicit verdict phrases count as prose evidence, quoted spans are blanked in one linear pass instead of rescanning from the start for every occurrence, and the wrapper prefixes match the hard-refusal parser. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 84 +++++++++---------- ...uter-combo-failover-classification.test.ts | 29 +++++++ 2 files changed, 71 insertions(+), 42 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 11f247147e1..2d7263e8b09 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode } from "../lib/errors"; +import { classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; import { isHardPolicyOrOriginRefusal } from "../lib/hard-refusal"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; @@ -1015,49 +1015,34 @@ function isReflectedValidationText(text: string): boolean { } /** - * A moderation phrase echoed inside quotes is a reflected parameter value (`Invalid enum value - * 'blocked by moderation'`), not a verdict. Track quote state so verdict evidence must appear - * in prose; an apostrophe inside a word ("don't") is literal text, not a quote delimiter. + * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value + * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass + * blanks every quoted span; an apostrophe inside a word ("don't") is literal text, not a quote. */ -function isInsideQuotedSpan(text: string, end: number): boolean { - let inSingle = false; - let inDouble = false; - for (let i = 0; i < end; i += 1) { - const ch = text[i]; +function unquotedProse(text: string): string { + let out = ""; + let quote: "'" | "\"" | null = null; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]!; if (ch === "\\") { + out += quote ? " " : ch + (text[i + 1] ?? ""); i += 1; continue; } - if (inDouble) { - if (ch === "\"") inDouble = false; + if (quote) { + if (ch === quote) quote = null; + out += " "; continue; } - if (inSingle) { - if (ch === "'") inSingle = false; + if (ch === "\"" || (ch === "'" && !(i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? "")))) { + quote = ch; + out += " "; continue; } - if (ch === "\"") { - inDouble = true; - continue; - } - if ( - ch === "'" - && !(i > 0 && i + 1 < text.length && /[A-Za-z]/.test(text[i - 1]) && /[A-Za-z]/.test(text[i + 1])) - ) { - inSingle = true; - } - } - return inSingle || inDouble; -} - -function hasUnquotedOccurrence(text: string, needle: string): boolean { - let from = 0; - for (;;) { - const idx = text.indexOf(needle, from); - if (idx === -1) return false; - if (!isInsideQuotedSpan(text, idx)) return true; - from = idx + 1; + out += ch; } + // An unterminated quote leaves its tail blanked, matching the old "inside a quote" verdict. + return out; } const MODERATION_REFUSAL_VERDICT_PHRASES = [ @@ -1067,13 +1052,20 @@ const MODERATION_REFUSAL_VERDICT_PHRASES = [ "content management policy", ]; +/** + * Any hard-refusal or non-replayable code anywhere in the text, structured or serialized. A + * moderation verdict never outranks one: the client must not retry these, and a body that + * carries both is not evidence that another target may serve the request. + */ +const HARD_STOP_TOKEN = + /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; + function namesModerationRefusal(text: string): boolean { if (isReflectedValidationText(text)) return false; - const lower = text.toLowerCase(); - for (const phrase of MODERATION_REFUSAL_VERDICT_PHRASES) { - if (hasUnquotedOccurrence(lower, phrase)) return true; - } - return hasUnquotedOccurrence(lower, "moderation") && hasUnquotedOccurrence(lower, "categories:"); + // Only an explicit verdict phrase counts. A validation error can name moderation categories + // ("Invalid metadata value: moderation categories: harassment") without any refusal. + const prose = unquotedProse(text.toLowerCase()); + return MODERATION_REFUSAL_VERDICT_PHRASES.some(phrase => prose.includes(phrase)); } /** @@ -1091,11 +1083,18 @@ export function isProviderModerationRefusal( const normalizedCode = normalizedFailureCode(code); if ( isCyberPolicyCode(normalizedCode) + || isTerminalRefusalCode(normalizedCode) || isNonReplayableUpstreamCode(normalizedCode) || normalizedCode === "origin_rejected" ) { return false; } + // Hard refusals anywhere in the body outrank a moderation verdict, including nested codes, + // serialized wrappers and safety copy. The terminal refusal path owns those bodies. + if (message && (HARD_STOP_TOKEN.test(message) || isHardPolicyOrOriginRefusal(message) + || safetyRefusalCodeFromMessage(message) !== undefined)) { + return false; + } if (!message) { return normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; } @@ -1106,9 +1105,10 @@ export function isProviderModerationRefusal( let moderationCodeSeen = normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; for (let depth = 0; depth < 3; depth += 1) { - if (text.startsWith("Provider error 400: ") || text.startsWith("Provider error 422: ")) { - text = text.slice(text.indexOf(": ") + 2).trim(); - } + // The same wrapper prefixes the hard-refusal parser strips (src/lib/hard-refusal.ts). + const prefix = ["Provider error 400: ", "Provider error 422: ", "Provider error: ", "Gateway error 400: ", "Gateway error: "] + .find(candidate => text.startsWith(candidate)); + if (prefix) text = text.slice(prefix.length).trim(); if (text.startsWith("data:") && !text.includes("\n")) { text = text.slice("data:".length).trim(); } diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index e8fd1be41f5..da39774000f 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1607,3 +1607,32 @@ describe("provider moderation refusal", () => { }); }); + +describe("provider moderation refusal never outranks a hard refusal", () => { + const body = (error: Record, root: Record = {}) => + JSON.stringify({ ...root, error: { type: "invalid_request_error", ...error } }); + const cases: Array<[string, string, string | undefined, string | undefined]> = [ + ["invalid_prompt with content-policy prose", body({ code: "invalid_prompt", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy." }), "invalid_prompt", undefined], + ["bio_policy with moderation prose", body({ code: "bio_policy", message: "The input was refused by moderation. Categories: biology." }), "bio_policy", undefined], + ["misalignment code with moderation prose", body({ code: "misalignment_policy_violation", message: "The input was flagged by moderation." }), "misalignment_policy_violation", undefined], + ["content_filter code with safety-system prose", body({ code: "content_filter", message: "This request was blocked by our safety systems." }), "content_filter", undefined], + ["plain moderation text that also carries safety copy", "The input was refused by moderation. We've limited access to this content for safety reasons.", undefined, undefined], + ["terminal root code over a nested moderation code", body({ code: "content_filter", message: "The input was refused by moderation." }, { code: "invalid_prompt" }), "content_filter", "invalid_prompt"], + ["nested hard code beside a moderation code", body({ code: "content_filter", message: "Blocked.", innererror: { code: "bio_policy" } }), "content_filter", undefined], + ["unstripped gateway prefix with a non-replayable root", 'Gateway error 400: {"code":"upstream_no_response","error":{"code":"content_filter"}}', "content_filter", undefined], + ["validation error that merely names moderation categories", body({ message: "Invalid metadata value: moderation categories: harassment" }), undefined, undefined], + ]; + for (const [label, message, code, rootCode] of cases) { + test(label, () => { + expect(isProviderModerationRefusal(400, message, code)).toBe(false); + expect(comboFailureDecision(400, message, { code, rootCode })).toBe("stop"); + expect(comboFailureCooldownScope(400, message, { code })).not.toBe("none"); + }); + } + + test("the reported moderation refusal still hops without cooling the target", () => { + const message = body({ code: "moderation_refusal", param: "input", message: "The input was refused by moderation. Categories: harassment." }); + expect(comboFailureDecision(400, message, { code: "moderation_refusal" })).toBe("hop"); + expect(comboFailureCooldownScope(400, message, { code: "moderation_refusal" })).toBe("none"); + }); +}); From abbcb4dee7247d8a32cad477e41345f64755a631 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 02:02:29 +0900 Subject: [PATCH 09/29] fix(combo): keep root verdict codes terminal beside a moderation verdict Re-review follow-ups: - content_policy_violation, and every envelope-root verdict code dev keeps terminal (origin, content policy, context length, a generic invalid_request_error code), now stops even beside moderation prose. - An apostrophe between letters no longer closes a quoted echo, and backticks count as quotes, so echoed values cannot fake a verdict. - JSON-escaped nested hard codes are decoded before the token check. - structure/ and the combos guide state that a provider or gateway moderation verdict advances without cooling, and what still stops. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- docs-site/src/content/docs/guides/combos.md | 1 + src/combos/failover.ts | 23 ++++++++++++++----- structure/providers-and-adapters.md | 2 +- structure/transports/responses.md | 2 +- ...uter-combo-failover-classification.test.ts | 5 ++++ 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 2b22e3724b9..8ecb2254014 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -699,6 +699,7 @@ provider, see [Fallback off](/guides/fallback-off/). | HTTP 410 with an explicit model end-of-life, retired, deprecated, sunset, decommissioned, or no-longer-available signal | Cool that target and hop. Unrelated 410 responses remain terminal. | | Classified authentication, subscription, quota, rate-limit, overload, or upstream-server error | Cool the target and hop, even when the status alone is not sufficient. | | Confirmed context-window overflow before output, including HTTP 400 | Try the next untried eligible target without cooling this target; another model may have a larger window. Exhaustion returns the first confirmed HTTP 400 context-length error; a context refusal sent with another status still advances but is not retained. | +| HTTP 400 or 422 moderation verdict from the provider or a gateway (`moderation_refusal` or `content_filter` code, or "refused/blocked/flagged by moderation" prose) | Try the next target without cooling this one; another provider's filter may accept the prompt. A body that also carries a terminal refusal (cyber or bio policy, `invalid_prompt`, `content_policy_violation`, safety-system wording) still stops. | | Client cancellation (499), `origin_rejected`, cyber-policy refusal, or other invalid request | Stop and return the error; another target would not make the request valid. | | Exact HTTP 400 saying a model is not supported when using Codex with a ChatGPT account | Try the next declared target before output commitment. The message must be the whole refusal in `detail`, `error.message`, or bare text; competing `detail` and `error` envelopes stay terminal. This does not quarantine the account or cool every model on the provider. | | Structured HTTP 400 rejecting optional `user`, an unsupported reasoning effort, model-scoped image input, or input media the route cannot read (`unsupported_input_modality`) | For an effort refusal, try the next lower published rung on the same target while the send budget allows; if no rung works, hop before output commitment without cooling if the refusal matches a recognized compatibility shape, otherwise stay terminal. Other listed mismatches hop without cooling; see request-local target compatibility below. | diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 2d7263e8b09..a264497eb99 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1021,7 +1021,7 @@ function isReflectedValidationText(text: string): boolean { */ function unquotedProse(text: string): string { let out = ""; - let quote: "'" | "\"" | null = null; + let quote: string | null = null; for (let i = 0; i < text.length; i += 1) { const ch = text[i]!; if (ch === "\\") { @@ -1029,12 +1029,14 @@ function unquotedProse(text: string): string { i += 1; continue; } + // An apostrophe between letters ("it's") is literal both outside and inside a quote. + const inWordApostrophe = ch === "'" && i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); if (quote) { - if (ch === quote) quote = null; + if (ch === quote && !inWordApostrophe) quote = null; out += " "; continue; } - if (ch === "\"" || (ch === "'" && !(i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? "")))) { + if (ch === "\"" || ch === "`" || (ch === "'" && !inWordApostrophe)) { quote = ch; out += " "; continue; @@ -1058,7 +1060,12 @@ const MODERATION_REFUSAL_VERDICT_PHRASES = [ * carries both is not evidence that another target may serve the request. */ const HARD_STOP_TOKEN = - /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; + /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|content_policy_violation|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; + +/** JSON \\uXXXX escapes decoded, so an escaped nested code cannot slip past {@link HARD_STOP_TOKEN}. */ +function withJsonEscapesDecoded(text: string): string { + return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); +} function namesModerationRefusal(text: string): boolean { if (isReflectedValidationText(text)) return false; @@ -1085,13 +1092,15 @@ export function isProviderModerationRefusal( isCyberPolicyCode(normalizedCode) || isTerminalRefusalCode(normalizedCode) || isNonReplayableUpstreamCode(normalizedCode) - || normalizedCode === "origin_rejected" + // The envelope-root verdict codes (origin, content policy, context length, a generic + // invalid_request_error code) stop on their own; a moderation phrase beside one cannot hop. + || rootCodeStatesVerdict(normalizedCode) ) { return false; } // Hard refusals anywhere in the body outrank a moderation verdict, including nested codes, // serialized wrappers and safety copy. The terminal refusal path owns those bodies. - if (message && (HARD_STOP_TOKEN.test(message) || isHardPolicyOrOriginRefusal(message) + if (message && (HARD_STOP_TOKEN.test(withJsonEscapesDecoded(message)) || isHardPolicyOrOriginRefusal(message) || safetyRefusalCodeFromMessage(message) !== undefined)) { return false; } @@ -1166,8 +1175,10 @@ export function isProviderModerationRefusal( for (const c of [rootCode, errCode]) { if ( isCyberPolicyCode(c) + || isTerminalRefusalCode(c) || isNonReplayableUpstreamCode(c) || c === "origin_rejected" + || c === "content_policy_violation" ) { return false; } diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 423de96bbd2..343fddd90e6 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`: a `moderation_refusal` or `content_filter` code, or an unquoted verdict phrase such as "refused by moderation"): that provider's filter is target-local, so the prompt goes to the next combo target. It never outranks a stop: any terminal refusal or root verdict code, non-replayable code, hard-refusal token (also JSON-escaped) or safety copy anywhere in the body keeps the terminal decision and its cooldown. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/structure/transports/responses.md b/structure/transports/responses.md index 8f75fe93644..eef2f801d5c 100644 --- a/structure/transports/responses.md +++ b/structure/transports/responses.md @@ -597,4 +597,4 @@ status-only fallback. Rebuilt failures remain non-replayable and cyber-policy fa `Retry-After` nor quota-reset metadata. The [Responses failover contract](responses-failover.md) owns the bounded recovery and replay decisions. -On the `responses` inbound wire only (`terminalRefusalAsFailed`, set by the same test as `enforceDeclaredToolNames` in `src/server/responses/adapter-delivery.ts`, `src/server/responses/run-turn-execution.ts` and the sidecar loops started from `src/server/responses/sidecar-execution.ts`), an adapter `incomplete` with `reason: "content_filter"` and `retryable: false` is delivered as `response.failed` whose `error` and `last_error` carry `invalid_request_error` / `invalid_prompt` with `retryable: false` and the turn's usage, built by `src/bridge/terminal-refusal.ts`: Codex retries every `response.incomplete` without reading `incomplete_details.retryable` and ends a turn only on a terminal-refusal `error.code` (#4312). The Anthropic and Kiro adapters emit that event today. Inside a combo a refusal with no output never reaches the client as a stream: `src/server/responses/combo-stream-preflight.ts` converts the zero-output `response.failed` into an HTTP 400 JSON error with the same `invalid_prompt` code, and `comboFailureDecision` stops the chain on it instead of sending the refused prompt to the next target (`tests/routing/combo-terminal-refusal.test.ts`). The web-search and image sidecar loops (`src/web-search/loop.ts`, `src/images/loop.ts`) check `isTerminalRefusalIncomplete` before looping, on every inbound wire: a refused iteration that carried a sidecar call ends the turn instead of running the call and re-asking the model with the refused context (`tests/web-search/web-search-terminal-refusal.test.ts`, `tests/images/loop-terminal-refusal.test.ts`); `src/web-search/run-turn-loop.ts` already stops on any incomplete. Every other incomplete, the chat and Anthropic inbound wires, and `buildResponseJSON` keep the incomplete shape. A `response.failed` built from an adapter error carries `retryable: false` for every code in `isTerminalRefusalCode` (`src/lib/errors.ts`), not only cyber policy. `tests/adapters/bridge-terminal-refusal.test.ts` pins both, `tests/server/server-terminal-refusal-wiring.test.ts` pins which inbound wire opts in, and `tests/server/server-kiro-refusal-e2e.test.ts` pins the adapter, runTurn and both sidecar-loop deliveries. +On the `responses` inbound wire only (`terminalRefusalAsFailed`, set by the same test as `enforceDeclaredToolNames` in `src/server/responses/adapter-delivery.ts`, `src/server/responses/run-turn-execution.ts` and the sidecar loops started from `src/server/responses/sidecar-execution.ts`), an adapter `incomplete` with `reason: "content_filter"` and `retryable: false` is delivered as `response.failed` whose `error` and `last_error` carry `invalid_request_error` / `invalid_prompt` with `retryable: false` and the turn's usage, built by `src/bridge/terminal-refusal.ts`: Codex retries every `response.incomplete` without reading `incomplete_details.retryable` and ends a turn only on a terminal-refusal `error.code` (#4312). The Anthropic and Kiro adapters emit that event today. Inside a combo a refusal with no output never reaches the client as a stream: `src/server/responses/combo-stream-preflight.ts` converts the zero-output `response.failed` into an HTTP 400 JSON error with the same `invalid_prompt` code, and `comboFailureDecision` stops the chain on it instead of sending the refused prompt to the next target (`tests/routing/combo-terminal-refusal.test.ts`). That covers a model's own refusal; a provider or gateway moderation verdict on HTTP 400/422 is target-local and may advance (see `isProviderModerationRefusal` in [Combo model refusal evidence](../providers-and-adapters.md#combo-model-refusal-evidence)). The web-search and image sidecar loops (`src/web-search/loop.ts`, `src/images/loop.ts`) check `isTerminalRefusalIncomplete` before looping, on every inbound wire: a refused iteration that carried a sidecar call ends the turn instead of running the call and re-asking the model with the refused context (`tests/web-search/web-search-terminal-refusal.test.ts`, `tests/images/loop-terminal-refusal.test.ts`); `src/web-search/run-turn-loop.ts` already stops on any incomplete. Every other incomplete, the chat and Anthropic inbound wires, and `buildResponseJSON` keep the incomplete shape. A `response.failed` built from an adapter error carries `retryable: false` for every code in `isTerminalRefusalCode` (`src/lib/errors.ts`), not only cyber policy. `tests/adapters/bridge-terminal-refusal.test.ts` pins both, `tests/server/server-terminal-refusal-wiring.test.ts` pins which inbound wire opts in, and `tests/server/server-kiro-refusal-e2e.test.ts` pins the adapter, runTurn and both sidecar-loop deliveries. diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index da39774000f..4f08f16f420 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1621,6 +1621,11 @@ describe("provider moderation refusal never outranks a hard refusal", () => { ["nested hard code beside a moderation code", body({ code: "content_filter", message: "Blocked.", innererror: { code: "bio_policy" } }), "content_filter", undefined], ["unstripped gateway prefix with a non-replayable root", 'Gateway error 400: {"code":"upstream_no_response","error":{"code":"content_filter"}}', "content_filter", undefined], ["validation error that merely names moderation categories", body({ message: "Invalid metadata value: moderation categories: harassment" }), undefined, undefined], + ["content_policy_violation root over a nested moderation code", body({ code: "content_filter", message: "The input was refused by moderation." }, { code: "content_policy_violation" }), "content_filter", "content_policy_violation"], + ["content_policy_violation code with moderation prose", body({ code: "content_policy_violation", message: "Your request was refused by moderation." }), "content_policy_violation", undefined], + ["an echoed value whose quote holds an apostrophe", body({ message: "Invalid enum value 'it's refused by moderation' for field mode" }), undefined, undefined], + ["an echoed value in backticks", body({ message: "Unknown tool name `refused by moderation` in tools[0]" }), undefined, undefined], + ["a JSON-escaped nested hard code", '{"error":{"code":"content_filter","message":"refused by moderation","inner":{"code":"\\u0062io_policy"}}}', "content_filter", undefined], ]; for (const [label, message, code, rootCode] of cases) { test(label, () => { From 3ae4fc37649b7026f5d772878f30d886d28999ab Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:31:58 +0900 Subject: [PATCH 10/29] fix(combo): read the parsed root verdict when only the nested code is passed Final-review follow-ups: - The policy-fallback path passes error.code alone, so a body whose root states context_length_exceeded, invalid_request_error or a client cancellation beside a nested content_filter hopped. The parsed envelope root now stops through rootCodeStatesVerdict as the combo path does. - Typographic quotes (curly double and single, guillemets, low-high) are quote delimiters, and a curly apostrophe inside a word stays literal. - The ko, ja and zh-cn combos guides carry the moderation row. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- .../src/content/docs/ja/guides/combos.md | 1 + .../src/content/docs/ko/guides/combos.md | 1 + .../src/content/docs/zh-cn/guides/combos.md | 1 + src/combos/failover.ts | 19 +++++++++++++++---- ...uter-combo-failover-classification.test.ts | 6 ++++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs-site/src/content/docs/ja/guides/combos.md b/docs-site/src/content/docs/ja/guides/combos.md index ecd110db6bb..9e0b0552089 100644 --- a/docs-site/src/content/docs/ja/guides/combos.md +++ b/docs-site/src/content/docs/ja/guides/combos.md @@ -128,6 +128,7 @@ ocx combo set balanced \ | HTTP 401、403、404、408、429、または任意の 5xx |ターゲットを冷却し、次の適格なターゲットに移動します。 | |モデルのサポート終了、retired、deprecated、sunset、decommissioned、または利用不可を明示する HTTP 410 |そのターゲットをクールダウンし、次へ進みます。無関係な 410 はターミナル エラーのままです。 | |機密認証、サブスクリプション、クォータ、レート制限、過負荷、またはアップストリーム サーバー エラー |ステータスだけでは物足りない場合でもターゲットを冷やしてホップさせましょう。 | +| プロバイダーまたはゲートウェイの HTTP 400 / 422 モデレーション判定（`moderation_refusal` または `content_filter` コード、あるいは "refused/blocked/flagged by moderation" の文言） |このターゲットを冷却せずに次のターゲットを試行します。別のプロバイダーのフィルターはプロンプトを受け入れる可能性があります。同じ本文に終了扱いの拒否（サイバーまたはバイオポリシー、`invalid_prompt`、`content_policy_violation`、安全システムの文言）が含まれる場合は停止します。 | |クライアントのキャンセル (499)、`origin_rejected`、サイバー ポリシーの拒否、またはその他の無効なリクエスト |停止してエラーを返します。別のターゲットではリクエストは有効になりません。 | | 出力前に確認したコンテキスト超過（HTTP 400 を含む） | クールダウンせず次の未試行の適格な対象へ進み、使い切った場合は最初に確認した HTTP 400 のコンテキストエラーを返します（他のステータスのコンテキスト拒否も次へ進みますが、保持されません）。 | | ChatGPT アカウントで Codex を使う際に、そのモデルをサポートしないことを示す正確な HTTP 400 の拒否 | 出力を確定する前に、次の設定済みターゲットを試します。拒否文全体が `detail`、`error.message`、または本文そのものにある場合に限ります。`detail` と `error` が競合する応答は終了扱いです。この拒否だけでアカウントを再認証待ちにしたり、プロバイダーの全モデルをクールダウンしたりしません。 | diff --git a/docs-site/src/content/docs/ko/guides/combos.md b/docs-site/src/content/docs/ko/guides/combos.md index e9a9e2c116e..ebb4e6e2c76 100644 --- a/docs-site/src/content/docs/ko/guides/combos.md +++ b/docs-site/src/content/docs/ko/guides/combos.md @@ -134,6 +134,7 @@ ocx combo set balanced \ | HTTP 401, 403, 404, 408, 429, 또는 모든 5xx | 대상을 쿨다운으로 보내고 다음 적합한 대상으로 넘어갑니다. | | 모델 수명 종료, retired, deprecated, sunset, decommissioned, 또는 더 이상 사용할 수 없다는 신호가 명시된 HTTP 410 | 해당 대상만 쿨다운으로 보내고 다음 대상으로 넘어갑니다. 관련 없는 410은 종결 오류로 유지합니다. | | 인증, 구독, 쿼터, 속도 제한, 과부하, 또는 상위 서버 오류로 분류됨 | 상태 코드만으로는 충분하지 않더라도 대상을 쿨다운으로 보내고 넘어갑니다. | +| 제공자나 게이트웨이의 HTTP 400 또는 422 moderation 판정(`moderation_refusal` 또는 `content_filter` 코드, 또는 "refused/blocked/flagged by moderation" 문구) | 이 대상을 쿨다운하지 않고 다음 대상을 시도합니다. 다른 제공자의 필터는 프롬프트를 받아들일 수 있습니다. 같은 본문에 종결 거부(cyber 또는 bio 정책, `invalid_prompt`, `content_policy_violation`, 안전 시스템 문구)가 함께 있으면 계속 멈춥니다. | | 클라이언트 취소(499), `origin_rejected`, cyber-policy refusal, 또는 기타 invalid request | 멈추고 오류를 반환합니다. 다른 대상을 써도 요청이 유효해지지 않기 때문입니다. | | 출력 전 확인된 컨텍스트 초과 오류(HTTP 400 포함) | 쿨다운 없이 다음 미시도 적격 대상을 시도합니다. 모두 실패하면 처음 확인된 HTTP 400 컨텍스트 오류를 반환합니다(다른 상태 코드의 컨텍스트 거부도 다음 대상으로 넘어가지만 보관되지 않습니다). | | ChatGPT 계정으로 Codex를 사용할 때 해당 모델을 지원하지 않는다는 정확한 HTTP 400 거부 | 출력이 시작되기 전에 다음 설정 대상으로 넘어갑니다. 거부 문장 전체가 `detail`, `error.message` 또는 일반 텍스트에 있어야 하며, `detail`과 `error`가 충돌하면 중단합니다. 이 사유만으로 계정을 재인증 상태로 바꾸거나 공급자의 모든 모델에 쿨다운을 걸지는 않습니다. | diff --git a/docs-site/src/content/docs/zh-cn/guides/combos.md b/docs-site/src/content/docs/zh-cn/guides/combos.md index 6d72cf00f1a..05475bcf0db 100644 --- a/docs-site/src/content/docs/zh-cn/guides/combos.md +++ b/docs-site/src/content/docs/zh-cn/guides/combos.md @@ -154,6 +154,7 @@ combo 失败分为 **跳转** 失败和 **终止** 失败。 | HTTP 401、403、404、408、429，或任何 5xx | 使该目标进入冷却，并跳转到下一个合格目标。 | | HTTP 410，并明确表明模型已到生命周期终点、retired、deprecated、sunset、decommissioned 或不再可用 | 仅冷却该目标并继续跳转。无关的 410 仍然是终止错误。 | | 被分类为认证、订阅、配额、速率限制、过载或上游服务器错误 | 即使仅凭状态码不足以判断，也会使该目标进入冷却并跳转。 | +| 提供方或网关的 HTTP 400 或 422 审核判定（`moderation_refusal` 或 `content_filter` 代码，或 "refused/blocked/flagged by moderation" 文字） | 不冷却该目标，尝试下一个目标；其他提供方的过滤器可能接受该提示。若同一响应体中还带有终止性拒绝（cyber 或 bio 策略、`invalid_prompt`、`content_policy_violation`、安全系统文字），仍会停止。 | | 客户端取消（499）、`origin_rejected`、cyber-policy 拒绝、或其他无效请求 | 停止并返回错误；换其他目标也无法让请求变得有效。 | | 输出前确认的上下文超限（包括 HTTP 400） | 不冷却该目标，尝试下一个尚未尝试的合格目标；耗尽后返回第一个确认的 HTTP 400 上下文错误（其他状态码的上下文拒绝也会切换到下一个目标，但不会被保留）。 | | HTTP 400 明确表示使用 ChatGPT 账户运行 Codex 时不支持该模型 | 在输出提交前尝试下一个已配置目标。完整拒绝句必须位于 `detail`、`error.message` 或纯文本中；同时包含 `detail` 与 `error` 的混合封装仍会终止请求。此原因本身不会将账户标为需要重新登录，也不会让供应商的所有模型进入冷却。 | diff --git a/src/combos/failover.ts b/src/combos/failover.ts index a264497eb99..8d1286d9cd0 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1014,6 +1014,12 @@ function isReflectedValidationText(text: string): boolean { ); } +/** Opening quote characters and the character that closes each, ASCII and typographic. */ +const QUOTE_PAIRS: Record = { + "\"": "\"", "`": "`", + "\u201c": "\u201d", "\u201e": "\u201c", "\u2018": "\u2019", "\u00ab": "\u00bb", "\u2039": "\u203a", +}; + /** * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass @@ -1029,15 +1035,17 @@ function unquotedProse(text: string): string { i += 1; continue; } - // An apostrophe between letters ("it's") is literal both outside and inside a quote. - const inWordApostrophe = ch === "'" && i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); + // An apostrophe between letters ("it's", "it\u2019s") is literal both outside and inside a quote. + const inWordApostrophe = (ch === "'" || ch === "\u2019") && i > 0 + && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); if (quote) { if (ch === quote && !inWordApostrophe) quote = null; out += " "; continue; } - if (ch === "\"" || ch === "`" || (ch === "'" && !inWordApostrophe)) { - quote = ch; + const closing = ch === "'" ? (inWordApostrophe ? undefined : "'") : QUOTE_PAIRS[ch]; + if (closing !== undefined) { + quote = closing; out += " "; continue; } @@ -1186,6 +1194,9 @@ export function isProviderModerationRefusal( moderationCodeSeen = true; } } + // A root verdict code stops on its own even when the caller passed only the nested code + // (the policy-fallback path reads error.code alone), so read the parsed root here too. + if (rootCode !== "" && rootCodeStatesVerdict(rootCode)) return false; const errorObjectModerationCode = errCode === "moderation_refusal" || errCode === "content_filter"; if (typeof e.param === "string" && e.param) { diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 4f08f16f420..5234d804ac9 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1626,6 +1626,12 @@ describe("provider moderation refusal never outranks a hard refusal", () => { ["an echoed value whose quote holds an apostrophe", body({ message: "Invalid enum value 'it's refused by moderation' for field mode" }), undefined, undefined], ["an echoed value in backticks", body({ message: "Unknown tool name `refused by moderation` in tools[0]" }), undefined, undefined], ["a JSON-escaped nested hard code", '{"error":{"code":"content_filter","message":"refused by moderation","inner":{"code":"\\u0062io_policy"}}}', "content_filter", undefined], + // The policy-fallback path passes only error.code; the parsed root still speaks. + ...(["context_length_exceeded", "invalid_request_error", "client_closed_request", "client_cancelled"] as const).map(root => + [`a ${root} root read from the body when only the nested code is passed`, body({ code: "content_filter", message: "The input was refused by moderation." }, { code: root }), "content_filter", undefined] as [string, string, string | undefined, string | undefined]), + ["an echoed value in curly double quotes", body({ message: "Invalid value: \u201cblocked by moderation\u201d for messages[0].content", param: "messages[0].content" }), undefined, undefined], + ["an echoed value in guillemets", body({ message: "Invalid value: \u00abblocked by moderation\u00bb for messages[0].content", param: "messages[0].content" }), undefined, undefined], + ["an echoed value in curly single quotes holding an apostrophe", body({ message: "Invalid value: \u2018it\u2019s refused by moderation\u2019 for mode" }), undefined, undefined], ]; for (const [label, message, code, rootCode] of cases) { test(label, () => { From a10e64727d2e6a6907358c4a86e02c3d89c19430 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:05:48 +0900 Subject: [PATCH 11/29] fix(combo): judge a moderation hop on the whole body Fourth-review follow-ups: - The combo path classifies at most 500 characters of a failure body, so a hard refusal past that cut was invisible to the moderation gate and the body hopped. consumeComboFailure and the stream preflight now judge the complete bounded body and pass providerModerationRefusal, as they already do for the OpenRouter opaque-400 verdict; the decision's own code can still veto it. Without that verdict, a text of 500 characters or more never hops. - CJK corner brackets, white corner brackets, lenticular, fullwidth and double-prime quotes count as quotes, and the low single quote pairs correctly. - A passed invalid_request_error (the policy path's type fallback for a null code) no longer vetoes the canonical moderation refusal; as a parsed envelope root it still stops. - The Messages native preflight passes the payload root code. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 60 ++++++++++++++----- src/server/messages-native-preflight.ts | 1 + .../responses/combo-stream-preflight.ts | 6 +- src/server/responses/core-combo-failure.ts | 8 +++ src/server/responses/core-combo.ts | 2 + src/server/responses/core-options.ts | 2 + structure/providers-and-adapters.md | 2 +- ...uter-combo-failover-classification.test.ts | 52 ++++++++++++++-- 8 files changed, 111 insertions(+), 22 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 8d1286d9cd0..c8c149a1550 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1017,9 +1017,13 @@ function isReflectedValidationText(text: string): boolean { /** Opening quote characters and the character that closes each, ASCII and typographic. */ const QUOTE_PAIRS: Record = { "\"": "\"", "`": "`", - "\u201c": "\u201d", "\u201e": "\u201c", "\u2018": "\u2019", "\u00ab": "\u00bb", "\u2039": "\u203a", + "\u201c": "\u201d", "\u201e": "\u201c", "\u2018": "\u2019", "\u201a": "\u2018", "\u00ab": "\u00bb", "\u2039": "\u203a", + "\u300c": "\u300d", "\u300e": "\u300f", "\u3010": "\u3011", "\u301d": "\u301e", "\uff02": "\uff02", "\u2033": "\u2033", "\u2036": "\u2036", }; +/** The classification text the combo path bounds a body to (src/server/responses/core-errors.ts). */ +const MODERATION_EVIDENCE_LIMIT = 500; + /** * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass @@ -1089,23 +1093,40 @@ function namesModerationRefusal(text: string): boolean { * * It must precede the generic invalid_request_error stop. */ +/** Whether the caller's status and structured code leave room for a moderation hop at all. */ +function moderationCodeAllowsHop(status: number, normalizedCode: string): boolean { + if (status !== 400 && status !== 422) return false; + return !( + isCyberPolicyCode(normalizedCode) + || isTerminalRefusalCode(normalizedCode) + || isNonReplayableUpstreamCode(normalizedCode) + // The envelope-root verdict codes (origin, content policy, context length, cancellation) stop + // on their own; a moderation phrase beside one cannot hop. A passed invalid_request_error is + // usually the error type standing in for an absent code (policy-fallback reads it that way), + // so it is not a verdict here; as a parsed envelope root below it still is. + || (normalizedCode !== "invalid_request_error" && rootCodeStatesVerdict(normalizedCode)) + ); +} + +/** + * The moderation hop for one decision. A caller that read the complete bounded body (the combo + * failure reader, the stream preflight) passes its whole-body verdict; the structured code of + * this decision can still veto it. Without one, the possibly truncated message decides. + */ +function providerModerationHop(status: number, message: string, rawCode: string | null | undefined, wholeBody?: boolean): boolean { + if (wholeBody === undefined) return isProviderModerationRefusal(status, message, rawCode); + return wholeBody && moderationCodeAllowsHop(status, normalizedFailureCode(rawCode)); +} + export function isProviderModerationRefusal( status: number, message: string, code?: string | null, + /** True when `message` is the complete bounded body rather than a possibly cut classification text. */ + complete = false, ): boolean { - if (status !== 400 && status !== 422) return false; const normalizedCode = normalizedFailureCode(code); - if ( - isCyberPolicyCode(normalizedCode) - || isTerminalRefusalCode(normalizedCode) - || isNonReplayableUpstreamCode(normalizedCode) - // The envelope-root verdict codes (origin, content policy, context length, a generic - // invalid_request_error code) stop on their own; a moderation phrase beside one cannot hop. - || rootCodeStatesVerdict(normalizedCode) - ) { - return false; - } + if (!moderationCodeAllowsHop(status, normalizedCode)) return false; // Hard refusals anywhere in the body outrank a moderation verdict, including nested codes, // serialized wrappers and safety copy. The terminal refusal path owns those bodies. if (message && (HARD_STOP_TOKEN.test(withJsonEscapesDecoded(message)) || isHardPolicyOrOriginRefusal(message) @@ -1115,6 +1136,11 @@ export function isProviderModerationRefusal( if (!message) { return normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; } + // Hop only on evidence seen whole. The combo path classifies at most the first + // MODERATION_EVIDENCE_LIMIT characters of a body (normalizeUpstreamErrorText), so a text that + // long may be cut, and a hard refusal after the cut would be invisible to the gate above. The + // body readers pass the whole body with `complete` instead. + if (!complete && message.length >= MODERATION_EVIDENCE_LIMIT) return false; if (message.length > 32_768) return false; let text = message.trim(); @@ -1244,6 +1270,8 @@ export function comboFailureCooldownScope( code?: string | number | null; malformedResponseFormat?: boolean; openRouterOpaque400?: boolean; + /** Whole-body moderation verdict from a reader that saw the complete bounded body. */ + providerModerationRefusal?: boolean; }, ): ComboFailureCooldownScope { const rawCode = typeof options?.code === "number" ? String(options.code) : options?.code; @@ -1263,7 +1291,7 @@ export function comboFailureCooldownScope( // A capability gap says the target is healthy and the request did not fit it, which is the // same reason every other entry here refuses to cool a target. || isResponseFormatCapabilityRefusal(status, message, rawCode) - || isProviderModerationRefusal(status, message, rawCode) + || providerModerationHop(status, message, rawCode, options?.providerModerationRefusal) ) return "none"; if (isProviderScopedQuotaCap(status, message, rawCode)) return "provider"; // A rejected or unpaid credential is provider-wide evidence: every target that routes @@ -1407,6 +1435,8 @@ export function comboFailureDecision( verifiedPlanRefusal?: boolean; malformedResponseFormat?: boolean; openRouterOpaque400?: boolean; + /** Whole-body moderation verdict from a reader that saw the complete bounded body. */ + providerModerationRefusal?: boolean; }, ): ComboFailureDecision { const rootRawCode = typeof options?.rootCode === "number" ? String(options.rootCode) : options?.rootCode; @@ -1433,6 +1463,8 @@ function decideComboFailure( verifiedPlanRefusal?: boolean; malformedResponseFormat?: boolean; openRouterOpaque400?: boolean; + /** Whole-body moderation verdict from a reader that saw the complete bounded body. */ + providerModerationRefusal?: boolean; }, ): ComboFailureDecision { if (status === 499) return "stop"; @@ -1521,7 +1553,7 @@ function decideComboFailure( // reaches that list and returns terminal before anything can ask whether the next target // could have served the request (#4903). if (isResponseFormatCapabilityRefusal(status, message, rawCode)) return "hop"; - if (isProviderModerationRefusal(status, message, rawCode)) return "hop"; + if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal)) return "hop"; if (isAnthropicBillingFailure(status, message, rawCode)) return "hop"; // Kiro's body validator rejects the request for the identity it was routed under. The account // layer rotates logins first; this is the no-alternate case, so the next combo target is the diff --git a/src/server/messages-native-preflight.ts b/src/server/messages-native-preflight.ts index 94714bcf1fb..593944bf745 100644 --- a/src/server/messages-native-preflight.ts +++ b/src/server/messages-native-preflight.ts @@ -99,6 +99,7 @@ export async function preflightNativeMessagesResponse( if (logCtx.terminalSource !== "synthetic" && logCtx.errorCode !== UNDECLARED_TOOL_CALL_ERROR_CODE && comboFailureDecision(status, typeof error.message === "string" ? error.message : "", { code: typeof error.code === "string" ? error.code : undefined, + rootCode: typeof payload.code === "string" ? payload.code : undefined, }) === "hop") { failure = error; failureStatus = status; } else committed = true; } else if (payload.type === "message_stop") terminal = true; diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index edae9c01f84..b82fd451f15 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -6,6 +6,7 @@ import { hasConflictingCodexModelRefusalEnvelopes, isMalformedResponseFormatRecord, isOpenRouterOpaque400EnvelopeRoot, + isProviderModerationRefusal, isOpenRouterOpaque400Record, type CodexAccountModelRefusal, } from "../../combos/failover"; @@ -157,12 +158,15 @@ function bareErrorIsRetryable(payload: unknown): boolean { }; const malformedResponseFormat = status === 400 ? isMalformedResponseFormatRecord(rawRecord, message) : undefined; const openRouterOpaque400 = status === 400 ? isOpenRouterOpaque400EnvelopeRoot(event) : undefined; + // The parsed event is the whole terminal record, so its moderation verdict is not cut short. + const providerModerationRefusal = status === 400 || status === 422 + ? isProviderModerationRefusal(status, JSON.stringify(event), undefined, true) : undefined; // A conflicting root and record code hop only when each would hop on its own, so a root hard // stop is never hidden behind a retryable record code (and a retryable pair keeps dev's hop). return comboFailureDecision(status, message, { code: recordCode ?? rootCode, rootCode, codexModelRefusal: terminalModelRefusal(status, event), - malformedResponseFormat, openRouterOpaque400, + malformedResponseFormat, openRouterOpaque400, providerModerationRefusal, }) === "hop"; } diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index 82955bbce20..b6753b16b1c 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -8,6 +8,7 @@ import { codexAccountModelRefusalHardStopCode, isMalformedResponseFormatRequest, isOpenRouterOpaque400Envelope, + isProviderModerationRefusal, upstreamEnvelopeRootCode, type CodexAccountModelRefusal, } from "../../combos/failover"; @@ -66,6 +67,7 @@ export async function consumeComboFailure( let codexModelRefusal: CodexAccountModelRefusal | undefined; let malformedResponseFormat: boolean | undefined; let openRouterOpaque400: boolean | undefined; + let providerModerationRefusal: boolean | undefined; let usage: OcxUsage | undefined; let upstreamCode: string | undefined; let upstreamRootCode: string | undefined; @@ -107,6 +109,11 @@ export async function consumeComboFailure( malformedResponseFormat = isMalformedResponseFormatRequest(response.status, body.text); openRouterOpaque400 = isOpenRouterOpaque400Envelope(body.text); } + // Judged on the whole body: the classification text below is cut at 500 characters, and a + // hard refusal after the cut must still veto a moderation hop. + if (response.status === 400 || response.status === 422) { + providerModerationRefusal = isProviderModerationRefusal(response.status, body.text, undefined, true); + } // Status-independent: the nested record is what normalization selects, so a root verdict // beside it has to be captured here or the conflict rule never sees it. upstreamRootCode = upstreamEnvelopeRootCode(body.text); @@ -189,6 +196,7 @@ export async function consumeComboFailure( ...(codexModelRefusal !== undefined ? { codexModelRefusal } : {}), ...(malformedResponseFormat !== undefined ? { malformedResponseFormat } : {}), ...(openRouterOpaque400 !== undefined ? { openRouterOpaque400 } : {}), + ...(providerModerationRefusal !== undefined ? { providerModerationRefusal } : {}), ...(normalizedUpstreamCode !== undefined ? { upstreamCode: normalizedUpstreamCode } : {}), // A rewritten identity (policy, replay refusal) is the whole verdict; the root's own code // would only re-open a decision that rewrite already settled. diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index a340f386742..aff8b9e7541 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -1437,6 +1437,7 @@ export async function executeComboResponses( verifiedPlanRefusal: failure.verifiedPlanRefusal, malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, // The adapter the failed leg actually resolved: a configured `kiro` provider can route a // model through another wire via `modelAdapters`, and an unresolved leg grants no hop. failedAdapter: childLog.providerAdapter, @@ -1515,6 +1516,7 @@ export async function executeComboResponses( code: failure.upstreamCode, malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, }); const failedTargetKey = targetKey(pick.target); let failedTargetCooldownRecorded = false; diff --git a/src/server/responses/core-options.ts b/src/server/responses/core-options.ts index 976139eb7f8..1d7fd594d28 100644 --- a/src/server/responses/core-options.ts +++ b/src/server/responses/core-options.ts @@ -37,6 +37,8 @@ export interface ConsumedComboFailure { malformedResponseFormat?: boolean; /** Complete bounded-envelope OpenRouter opaque-400 verdict captured before display truncation. */ openRouterOpaque400?: boolean; + /** Complete bounded-envelope provider moderation verdict captured before display truncation. */ + providerModerationRefusal?: boolean; /** Structured upstream `error.code` when present in the failure body. */ upstreamCode?: string; /** diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 343fddd90e6..01265c2fc1d 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`: a `moderation_refusal` or `content_filter` code, or an unquoted verdict phrase such as "refused by moderation"): that provider's filter is target-local, so the prompt goes to the next combo target. It never outranks a stop: any terminal refusal or root verdict code, non-replayable code, hard-refusal token (also JSON-escaped) or safety copy anywhere in the body keeps the terminal decision and its cooldown. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`: a `moderation_refusal` or `content_filter` code, or an unquoted verdict phrase such as "refused by moderation"): that provider's filter is target-local, so the prompt goes to the next combo target. It never outranks a stop: any terminal refusal or root verdict code, non-replayable code, hard-refusal token (also JSON-escaped) or safety copy anywhere in the body keeps the terminal decision and its cooldown. The verdict needs the whole body: `consumeComboFailure` and the stream preflight judge the complete bounded body and pass `providerModerationRefusal`, and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 5234d804ac9..86ee919184e 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1270,9 +1270,12 @@ describe("provider moderation refusal", () => { const rawBody = JSON.stringify(moderationPayload); expect(isProviderModerationRefusal(400, rawBody)).toBe(true); for (const message of [rawBody, `Provider error 400: ${rawBody}`, `data: ${rawBody}`]) { - expect(comboFailureDecision(400, message)).toBe("hop"); - expect(comboFailureCooldownScope(400, message)).toBe("none"); + // The body readers judge the whole body; a wrapped copy is longer than the cut classification text. + const providerModerationRefusal = isProviderModerationRefusal(400, message, undefined, true); + expect(comboFailureDecision(400, message, { providerModerationRefusal })).toBe("hop"); + expect(comboFailureCooldownScope(400, message, { providerModerationRefusal })).toBe("none"); } + expect(comboFailureDecision(400, rawBody)).toBe("hop"); }); test("hops on various provider moderation phrases", () => { @@ -1485,8 +1488,9 @@ describe("provider moderation refusal", () => { // Pass through consumeComboFailure where production truncation occurs const response = new Response(raw, { status: 400 }); const failure = await consumeComboFailure(response); - expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); - expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("none"); + expect(failure.providerModerationRefusal).toBe(true); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); }); test("malformed JSON restricts recovery to error envelope and rejects non-error message fields", () => { @@ -1550,8 +1554,9 @@ describe("provider moderation refusal", () => { }); const response = new Response(raw, { status: 400 }); const failure = await consumeComboFailure(response); - expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("hop"); - expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode })).toBe("none"); + expect(failure.providerModerationRefusal).toBe(true); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); }); test("explicit content_filter code triggers failover even when param names other input field", () => { @@ -1647,3 +1652,38 @@ describe("provider moderation refusal never outranks a hard refusal", () => { expect(comboFailureCooldownScope(400, message, { code: "moderation_refusal" })).toBe("none"); }); }); + +describe("provider moderation refusal judged on the whole body", () => { + test("a hard refusal past the classification cut still stops the combo", async () => { + for (const hard of ["invalid_prompt", "bio_policy", "content_policy_violation", "upstream_no_response"]) { + const raw = JSON.stringify({ error: { + code: "content_filter", + message: "The input was refused by moderation.", + meta: { pad: "p".repeat(600), underlying: { code: hard } }, + } }); + const failure = await consumeComboFailure(new Response(raw, { status: 400 })); + expect(failure.classificationText.length).toBeLessThanOrEqual(500); + expect(failure.providerModerationRefusal).toBe(false); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal }; + expect({ hard, decision: comboFailureDecision(400, failure.classificationText, options) }).toEqual({ hard, decision: "stop" }); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + // Without the whole-body verdict, a text that may have been cut never hops either. + expect(isProviderModerationRefusal(400, raw.slice(0, 500), "content_filter")).toBe(false); + } + }); + + test("CJK and fullwidth quoted echoes are not verdicts", () => { + for (const [open, close] of [["\u300c", "\u300d"], ["\u300e", "\u300f"], ["\uff02", "\uff02"], ["\u201a", "\u2018"]]) { + const message = JSON.stringify({ error: { type: "invalid_request_error", message: "Invalid value " + open + "refused by moderation" + close + " for field mode" } }); + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + } + }); + + test("an error type passed as the code does not veto the canonical moderation refusal", () => { + // The policy-fallback path reads error.type when error.code is null. + const message = JSON.stringify({ error: { message: "The input was refused by moderation. Categories: harassment.", type: "invalid_request_error", param: null, code: null } }); + expect(comboFailureDecision(400, message, { code: "invalid_request_error" })).toBe("hop"); + expect(comboFailureDecision(400, message)).toBe("hop"); + }); +}); From 3baed003a1caffdeeb1f9bb9ddd8025e2539469a Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:24:57 +0900 Subject: [PATCH 12/29] fix(combo): keep provider cooldown and native-preflight parity for moderation Fifth-review follow-ups: - A moderation verdict no longer returns cooldown none ahead of quota and credential evidence: insufficient_quota or invalid_api_key beside moderation prose still cools the provider. - The Messages native preflight judges the whole error event like the HTTP reader and the stream preflight, so a root verdict code stops on every transport. - A backslash-escaped quoted echo counts as quoted. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 6 +++++- src/server/messages-native-preflight.ts | 4 ++++ .../messages-native-combo-framing.test.ts | 17 +++++++++++++++++ ...router-combo-failover-classification.test.ts | 15 +++++++++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index c8c149a1550..8c3cf7e5d1d 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1030,6 +1030,8 @@ const MODERATION_EVIDENCE_LIMIT = 500; * blanks every quoted span; an apostrophe inside a word ("don't") is literal text, not a quote. */ function unquotedProse(text: string): string { + // A backslash-escaped quote (a JSON string read raw) quotes exactly like the bare character. + text = text.replace(/\\(["'`])/g, "$1"); let out = ""; let quote: string | null = null; for (let i = 0; i < text.length; i += 1) { @@ -1291,13 +1293,15 @@ export function comboFailureCooldownScope( // A capability gap says the target is healthy and the request did not fit it, which is the // same reason every other entry here refuses to cool a target. || isResponseFormatCapabilityRefusal(status, message, rawCode) - || providerModerationHop(status, message, rawCode, options?.providerModerationRefusal) ) return "none"; if (isProviderScopedQuotaCap(status, message, rawCode)) return "provider"; // A rejected or unpaid credential is provider-wide evidence: every target that routes // through the same provider row carries the same key and will fail identically. if (status === 401 || status === 402 || status === 403) return "provider"; if (PROVIDER_SCOPED_FAILURE_CODES.has(code)) return "provider"; + // A moderation verdict is target-local, but only after quota and credential evidence had its + // say: a refusing key beside moderation prose still cools its provider. + if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal)) return "none"; // OpenRouter's opaque numeric-code wrapper may be target-specific (for example a gateway // rejecting a model's translated payload). It may advance the combo, but says nothing about // target health and must not quarantine the target. diff --git a/src/server/messages-native-preflight.ts b/src/server/messages-native-preflight.ts index 593944bf745..42de1326f30 100644 --- a/src/server/messages-native-preflight.ts +++ b/src/server/messages-native-preflight.ts @@ -1,4 +1,5 @@ import { comboFailureDecision } from "../combos"; +import { isProviderModerationRefusal } from "../combos/failover"; import { httpStatusFromTerminalError } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; import { sseFieldValue } from "../lib/sse-decoder"; @@ -100,6 +101,9 @@ export async function preflightNativeMessagesResponse( && comboFailureDecision(status, typeof error.message === "string" ? error.message : "", { code: typeof error.code === "string" ? error.code : undefined, rootCode: typeof payload.code === "string" ? payload.code : undefined, + // The whole event, as the HTTP and stream readers judge it, so transports cannot disagree. + providerModerationRefusal: status === 400 || status === 422 + ? isProviderModerationRefusal(status, JSON.stringify(payload), undefined, true) : undefined, }) === "hop") { failure = error; failureStatus = status; } else committed = true; } else if (payload.type === "message_stop") terminal = true; diff --git a/tests/claude-integration/messages-native-combo-framing.test.ts b/tests/claude-integration/messages-native-combo-framing.test.ts index 1435b2e32a3..624abf1cf56 100644 --- a/tests/claude-integration/messages-native-combo-framing.test.ts +++ b/tests/claude-integration/messages-native-combo-framing.test.ts @@ -114,3 +114,20 @@ for (const [name, block, failed] of [ else await result.response.body?.cancel(); }); } + +test("native combo preflight judges a moderation refusal on the whole event, like the HTTP reader", async () => { + const run = async (event: Record) => { + const raw = prelude + "event: error\n" + "data: " + JSON.stringify(event) + "\n\n"; + const result = await preflightNativeMessagesResponse(new Response(raw, { + headers: { "content-type": "text/event-stream" }, + }), { model: "fixture", provider: "anth" }, createTestTranslatorBudget()); + await result.response.body?.cancel(); + return result.kind; + }; + const error = { type: "invalid_request_error", code: "content_filter", message: "The input was refused by moderation." }; + // A provider moderation verdict is target-local and advances the combo. + expect(await run({ type: "error", error })).toBe("failed"); + // A root verdict code beside it stops, as consumeComboFailure and the stream preflight decide. + expect(await run({ type: "error", code: "invalid_request_error", error })).toBe("accepted"); + expect(await run({ type: "error", code: "cyber_policy", error })).toBe("accepted"); +}); diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 86ee919184e..0266050ab10 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1687,3 +1687,18 @@ describe("provider moderation refusal judged on the whole body", () => { expect(comboFailureDecision(400, message)).toBe("hop"); }); }); + +describe("provider moderation refusal keeps provider evidence and escaped echoes", () => { + test("quota or credential codes beside moderation prose still cool the provider", () => { + for (const code of ["insufficient_quota", "invalid_api_key"]) { + const message = JSON.stringify({ error: { code, message: "The input was refused by moderation." } }); + expect({ code, scope: comboFailureCooldownScope(400, message, { code }) }).not.toEqual({ code, scope: "none" }); + } + }); + + test("a backslash-escaped quoted echo is not a verdict", () => { + const message = 'Invalid value: \\"refused by moderation\\" for field mode'; + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + }); +}); From 9ed7dc5da217e44c4346b491d94c064f30267b37 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:27:31 +0900 Subject: [PATCH 13/29] fix(combo): stop on type-only verdicts and nested quoted echoes beside moderation Sixth-review follow-ups: - client_closed_request, client_cancelled and context_length_exceeded join the hard-stop tokens, so a verdict stated only as a root or error type stops beside moderation prose, on the combo and policy paths alike. - Quoted spans track their own closer: an escaped quote inside a span is content, an escaped quote outside one opens a span its escaped twin closes, and a doubled delimiter closes only on the same doubled form. - The proxy's own cyber-policy fallback copy and every isCyberPolicyMessage form veto a moderation hop. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 64 +++++++++++++------ ...uter-combo-failover-classification.test.ts | 32 ++++++++++ 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 8c3cf7e5d1d..413535189f0 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; +import { CYBER_POLICY_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; import { isHardPolicyOrOriginRefusal } from "../lib/hard-refusal"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; @@ -1029,33 +1029,58 @@ const MODERATION_EVIDENCE_LIMIT = 500; * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass * blanks every quoted span; an apostrophe inside a word ("don't") is literal text, not a quote. */ +/** An apostrophe between letters ("it's", "it’s") is literal text, never a quote delimiter. */ +function isInWordApostrophe(text: string, i: number): boolean { + const ch = text[i]; + return (ch === "'" || ch === "\u2019") && i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); +} + function unquotedProse(text: string): string { - // A backslash-escaped quote (a JSON string read raw) quotes exactly like the bare character. - text = text.replace(/\\(["'`])/g, "$1"); let out = ""; - let quote: string | null = null; - for (let i = 0; i < text.length; i += 1) { + // The text that ends the open quoted span: one character, a doubled delimiter, or an escaped quote. + let closer: string | null = null; + for (let i = 0; i < text.length;) { const ch = text[i]!; - if (ch === "\\") { - out += quote ? " " : ch + (text[i + 1] ?? ""); + if (closer !== null) { + if (text.startsWith(closer, i) && !(closer.length === 1 && isInWordApostrophe(text, i))) { + out += " ".repeat(closer.length); + i += closer.length; + closer = null; + continue; + } + // An escaped character inside a span is content, including an escaped quote of its own kind. + if (ch === "\\" && !closer.startsWith("\\")) { + out += " "; + i += 2; + continue; + } + out += " "; i += 1; continue; } - // An apostrophe between letters ("it's", "it\u2019s") is literal both outside and inside a quote. - const inWordApostrophe = (ch === "'" || ch === "\u2019") && i > 0 - && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); - if (quote) { - if (ch === quote && !inWordApostrophe) quote = null; - out += " "; + if (ch === "\\") { + // A backslash-escaped quote (a JSON string read raw) opens a span its escaped twin closes. + const escapedClose = QUOTE_PAIRS[text[i + 1] ?? ""] ?? (text[i + 1] === "'" ? "'" : undefined); + if (escapedClose !== undefined) { + closer = "\\" + escapedClose; + out += " "; + i += 2; + continue; + } + out += ch + (text[i + 1] ?? ""); + i += 2; continue; } - const closing = ch === "'" ? (inWordApostrophe ? undefined : "'") : QUOTE_PAIRS[ch]; - if (closing !== undefined) { - quote = closing; - out += " "; + const close = ch === "'" ? (isInWordApostrophe(text, i) ? undefined : "'") : QUOTE_PAIRS[ch]; + if (close !== undefined) { + // A doubled delimiter ("" or '') opens a span the same doubled delimiter closes. + closer = close === ch && text[i + 1] === ch ? ch + ch : close; + out += " ".repeat(closer.length); + i += closer.length; continue; } out += ch; + i += 1; } // An unterminated quote leaves its tail blanked, matching the old "inside a quote" verdict. return out; @@ -1074,7 +1099,7 @@ const MODERATION_REFUSAL_VERDICT_PHRASES = [ * carries both is not evidence that another target may serve the request. */ const HARD_STOP_TOKEN = - /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|content_policy_violation|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; + /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|content_policy_violation|context_length_exceeded|client_closed_request|client_cancelled|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; /** JSON \\uXXXX escapes decoded, so an escaped nested code cannot slip past {@link HARD_STOP_TOKEN}. */ function withJsonEscapesDecoded(text: string): string { @@ -1132,7 +1157,8 @@ export function isProviderModerationRefusal( // Hard refusals anywhere in the body outrank a moderation verdict, including nested codes, // serialized wrappers and safety copy. The terminal refusal path owns those bodies. if (message && (HARD_STOP_TOKEN.test(withJsonEscapesDecoded(message)) || isHardPolicyOrOriginRefusal(message) - || safetyRefusalCodeFromMessage(message) !== undefined)) { + || safetyRefusalCodeFromMessage(message) !== undefined || isCyberPolicyMessage(message) + || message.toLowerCase().includes(CYBER_POLICY_FALLBACK_MESSAGE.toLowerCase()))) { return false; } if (!message) { diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 0266050ab10..f7197ad8791 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1702,3 +1702,35 @@ describe("provider moderation refusal keeps provider evidence and escaped echoes expect(comboFailureDecision(400, message)).toBe("stop"); }); }); + +describe("provider moderation refusal: type-only verdicts, nested escapes and cyber copy", () => { + for (const verdict of ["client_closed_request", "client_cancelled", "context_length_exceeded"]) { + for (const where of ["root", "error"] as const) { + test(verdict + " stated as the " + where + " type stops", async () => { + const error = { code: "content_filter", message: "The input was refused by moderation.", ...(where === "error" ? { type: verdict } : {}) }; + const raw = JSON.stringify({ ...(where === "root" ? { type: verdict } : {}), error }); + const failure = await consumeComboFailure(new Response(raw, { status: 400 })); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal }; + expect(comboFailureDecision(400, failure.classificationText, options)).toBe("stop"); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + expect(isProviderModerationRefusal(400, raw, "content_filter")).toBe(false); + }); + } + } + + test("nested and doubled quoted echoes are not verdicts", () => { + for (const message of [ + String.raw`Invalid 'input': "it's a \"refused by moderation\" value".`, + 'Invalid value: ""refused by moderation"" for field mode', + ]) { + expect({ message, hop: isProviderModerationRefusal(400, message) }).toEqual({ message, hop: false }); + expect(comboFailureDecision(400, message)).toBe("stop"); + } + }); + + test("this proxy's own cyber-policy copy beside moderation prose stops", () => { + const message = JSON.stringify({ error: { code: "content_filter", message: "The input was refused by moderation. Request blocked by the upstream cybersecurity policy." } }); + expect(isProviderModerationRefusal(400, message, "content_filter")).toBe(false); + expect(comboFailureDecision(400, message, { code: "content_filter" })).toBe("stop"); + }); +}); From 0201c39fa4eae29d30b4e2d2e9f1f88d6cfc0899 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:18:17 +0900 Subject: [PATCH 14/29] fix(combo): veto a moderation hop on any spelling or classifier verdict Seventh-review follow-ups. Chasing token spellings one at a time does not converge, so the gate now closes the classes: - Hard-refusal tokens match in any spelling: JSON escapes decoded and hyphens, dots and whitespace folded to underscores, in any carrier (details, last_error, metadata, innererror, response.error). - The shared classifyError must see nothing more specific than a generic invalid request: prose it maps to quota or client cancellation, and the proxy's own terminal-refusal copy, stop beside a moderation verdict. - Closing-only typographic delimiters quote an echoed value. - Content-management-policy prose counts only beside a content_filter code. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 28 +++++++++--- structure/providers-and-adapters.md | 2 +- ...uter-combo-failover-classification.test.ts | 44 ++++++++++++++++++- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 413535189f0..b18f80e1fe5 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { CYBER_POLICY_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; +import { CYBER_POLICY_FALLBACK_MESSAGE, TERMINAL_REFUSAL_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; import { isHardPolicyOrOriginRefusal } from "../lib/hard-refusal"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; @@ -1019,6 +1019,8 @@ const QUOTE_PAIRS: Record = { "\"": "\"", "`": "`", "\u201c": "\u201d", "\u201e": "\u201c", "\u2018": "\u2019", "\u201a": "\u2018", "\u00ab": "\u00bb", "\u2039": "\u203a", "\u300c": "\u300d", "\u300e": "\u300f", "\u3010": "\u3011", "\u301d": "\u301e", "\uff02": "\uff02", "\u2033": "\u2033", "\u2036": "\u2036", + // A closing character used as an opener (\u201dvalue\u201d, \u00bbvalue\u00bb) still quotes. + "\u201d": "\u201d", "\u2019": "\u2019", "\u00bb": "\u00bb", "\u203a": "\u203a", "\u300d": "\u300d", "\u300f": "\u300f", }; /** The classification text the combo path bounds a body to (src/server/responses/core-errors.ts). */ @@ -1071,7 +1073,7 @@ function unquotedProse(text: string): string { i += 2; continue; } - const close = ch === "'" ? (isInWordApostrophe(text, i) ? undefined : "'") : QUOTE_PAIRS[ch]; + const close = isInWordApostrophe(text, i) ? undefined : ch === "'" ? "'" : QUOTE_PAIRS[ch]; if (close !== undefined) { // A doubled delimiter ("" or '') opens a span the same doubled delimiter closes. closer = close === ch && text[i + 1] === ch ? ch + ch : close; @@ -1090,7 +1092,6 @@ const MODERATION_REFUSAL_VERDICT_PHRASES = [ "refused by moderation", "blocked by moderation", "flagged by moderation", - "content management policy", ]; /** @@ -1099,7 +1100,16 @@ const MODERATION_REFUSAL_VERDICT_PHRASES = [ * carries both is not evidence that another target may serve the request. */ const HARD_STOP_TOKEN = - /\b(?:cyber_policy|origin_rejected|bio_policy|misalignment_policy_violation|invalid_prompt|content_policy_violation|context_length_exceeded|client_closed_request|client_cancelled|upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused)\b/i; + /(? { "The response was refused by moderation.", "Prompt was blocked by moderation.", "Input was flagged by moderation.", - "Request was filtered due to the prompt triggering content management policy.", ]; for (const phrase of phrases) { const body = JSON.stringify({ error: { message: phrase } }); @@ -1734,3 +1733,46 @@ describe("provider moderation refusal: type-only verdicts, nested escapes and cy expect(comboFailureDecision(400, message, { code: "content_filter" })).toBe("stop"); }); }); + +describe("provider moderation refusal: any spelling of a verdict, terminal prose, closing-only quotes", () => { + const verdict = "The input was refused by moderation. Categories: harassment."; + const carriers: Array<[string, (code: string) => Record]> = [ + ["details", code => ({ error: { message: verdict, details: { code } } })], + ["last_error", code => ({ error: { message: verdict }, last_error: { code } })], + ["metadata", code => ({ error: { message: verdict, metadata: { code } } })], + ["innererror", code => ({ error: { code: "content_filter", message: verdict, innererror: { code } } })], + ["response.error", code => ({ response: { error: { code, message: verdict } } })], + ]; + for (const hard of ["content-policy-violation", "client-cancelled", "context length exceeded", "bio.policy", "upstream-no-response"]) { + for (const [carrier, build] of carriers) { + test("a " + hard + " code in " + carrier + " stops", async () => { + const raw = JSON.stringify(build(hard)); + const failure = await consumeComboFailure(new Response(raw, { status: 400 })); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal }; + expect(comboFailureDecision(400, failure.classificationText, options)).toBe("stop"); + expect(isProviderModerationRefusal(400, raw)).toBe(false); + }); + } + } + + test("terminal prose beside a moderation verdict stops", () => { + for (const tail of ["The upstream refused this request.", "client closed request", "You exceeded your current quota, please check your plan."]) { + const message = verdict + " " + tail; + expect({ tail, hop: isProviderModerationRefusal(400, message) }).toEqual({ tail, hop: false }); + expect(comboFailureCooldownScope(400, message)).not.toBe("none"); + } + }); + + test("a closing-only delimiter still quotes an echoed value", () => { + for (const [open, close] of [["\u201d", "\u201d"], ["\u00bb", "\u00bb"], ["\u2019", "\u2019"], ["\u300d", "\u300d"]]) { + const message = "Invalid value: " + open + "blocked by moderation" + close + " for field mode"; + expect({ open, hop: isProviderModerationRefusal(400, message) }).toEqual({ open, hop: false }); + } + }); + + test("content-management-policy prose needs the content_filter code", () => { + const message = JSON.stringify({ error: { message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy." } }); + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(isProviderModerationRefusal(400, message, "content_filter")).toBe(true); + }); +}); From c36d2b54c291e131421aca39fc00cb9af53d13ad Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:56:12 +0900 Subject: [PATCH 15/29] fix(combo): close escaped-whitespace spellings and restore 422 moderation hops Eighth-review follow-ups: - JSON-escaped whitespace (\n, \t, \r) folds to underscores too, so a hard code after an escaped newline stops. - The classifier veto classifies as a 400: classifyError maps an untyped 422 to upstream_error, which had made every genuine 422 moderation refusal terminal. - Only moderation-compatible structured codes (moderation_refusal, content_filter, invalid_request_error) may stand beside a verdict; a rate-limit, quota or overload code keeps its own decision and cooldown. - The routing-profile path judges the whole body like the combo reader, so a long moderation body no longer stops there while it hops elsewhere. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 14 ++++++-- src/server/responses/policy-fallback.ts | 5 ++- ...uter-combo-failover-classification.test.ts | 33 +++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index b18f80e1fe5..2a6568614af 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1026,6 +1026,9 @@ const QUOTE_PAIRS: Record = { /** The classification text the combo path bounds a body to (src/server/responses/core-errors.ts). */ const MODERATION_EVIDENCE_LIMIT = 500; +/** Structured codes that can stand beside a moderation verdict without being evidence of their own. */ +const MODERATION_COMPATIBLE_CODES = new Set(["moderation_refusal", "content_filter", "invalid_request_error", "400", "422"]); + /** * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass @@ -1108,7 +1111,8 @@ const HARD_STOP_TOKEN = * Underscores count as separators, so a token nested in a longer identifier still matches. */ function namesHardStopToken(text: string): boolean { - return HARD_STOP_TOKEN.test(withJsonEscapesDecoded(text).replace(/[-.\s]+/g, "_")); + // A raw JSON body spells whitespace as \n, \t or \r; fold those escapes as well. + return HARD_STOP_TOKEN.test(withJsonEscapesDecoded(text).replace(/\\[ntr]/g, "_").replace(/[-.\s]+/g, "_")); } /** JSON \\uXXXX escapes decoded, so an escaped nested code cannot slip past {@link HARD_STOP_TOKEN}. */ @@ -1133,6 +1137,9 @@ function namesModerationRefusal(text: string): boolean { /** Whether the caller's status and structured code leave room for a moderation hop at all. */ function moderationCodeAllowsHop(status: number, normalizedCode: string): boolean { if (status !== 400 && status !== 422) return false; + // Any other structured code (rate limit, quota, overload, model scope...) is its own evidence + // with its own cooldown; moderation prose beside it does not turn it into a free hop. + if (normalizedCode !== "" && !MODERATION_COMPATIBLE_CODES.has(normalizedCode)) return false; return !( isCyberPolicyCode(normalizedCode) || isTerminalRefusalCode(normalizedCode) @@ -1174,7 +1181,9 @@ export function isProviderModerationRefusal( || safetyRefusalCodeFromMessage(message) !== undefined || isCyberPolicyMessage(message) || lower.includes(CYBER_POLICY_FALLBACK_MESSAGE.toLowerCase()) || lower.includes(TERMINAL_REFUSAL_FALLBACK_MESSAGE.toLowerCase()) - || classifyError(status, "upstream_error", message).code !== "invalid_request_error")) { + // Classified as a 400: the moderation hop covers 422 too, and classifyError maps an untyped + // 422 to upstream_error rather than invalid_request_error. + || classifyError(400, "upstream_error", message).code !== "invalid_request_error")) { return false; } if (!message) { @@ -1257,6 +1266,7 @@ export function isProviderModerationRefusal( || isNonReplayableUpstreamCode(c) || c === "origin_rejected" || c === "content_policy_violation" + || (c !== "" && !MODERATION_COMPATIBLE_CODES.has(c)) ) { return false; } diff --git a/src/server/responses/policy-fallback.ts b/src/server/responses/policy-fallback.ts index 7db10d29f0f..1badfc77ee0 100644 --- a/src/server/responses/policy-fallback.ts +++ b/src/server/responses/policy-fallback.ts @@ -1,4 +1,4 @@ -import { comboFailureDecision } from "../../combos/failover"; +import { comboFailureDecision, isProviderModerationRefusal } from "../../combos/failover"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { isVerifiedModelNotInPlanResponse } from "../../lib/errors"; import { isShadowSourceModel } from "../../lib/shadow-call"; @@ -138,6 +138,9 @@ async function shouldHopPolicyCandidate(response: Response, signal?: AbortSignal code: upstreamCodeForComboFailure(response, errorCodeFromText(text)), // A passthrough candidate returns its upstream body unchanged, so the code is not provenance. verifiedPlanRefusal: isVerifiedModelNotInPlanResponse(response), + // The text is the whole bounded body, judged as the combo reader judges it. + ...(text && (response.status === 400 || response.status === 422) + ? { providerModerationRefusal: isProviderModerationRefusal(response.status, text, undefined, true) } : {}), }) === "hop"; } catch { return false; diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 9e1e75aa772..aeb9f33e091 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1776,3 +1776,36 @@ describe("provider moderation refusal: any spelling of a verdict, terminal prose expect(isProviderModerationRefusal(400, message, "content_filter")).toBe(true); }); }); + +describe("provider moderation refusal: escaped whitespace, 422, rate-limit codes, policy path", () => { + test("a JSON-escaped whitespace spelling of a hard code stops", async () => { + for (const sep of ["\n", "\t", "\r\n"]) { + for (const hard of ["content_policy_violation", "upstream_no_response", "client_cancelled", "client_closed_request"]) { + const raw = JSON.stringify({ error: { message: "The input was refused by moderation." + sep + hard } }); + const failure = await consumeComboFailure(new Response(raw, { status: 400 })); + expect({ hard, verdict: failure.providerModerationRefusal }).toEqual({ hard, verdict: false }); + expect(isProviderModerationRefusal(400, raw)).toBe(false); + } + } + }); + + test("a genuine 422 moderation refusal hops without cooling", () => { + for (const body of [ + { error: { code: "content_filter", message: "The input was refused by moderation." } }, + { error: { message: "The prompt was blocked by moderation. Categories: violence" } }, + ]) { + const message = JSON.stringify(body); + expect(comboFailureDecision(422, message)).toBe("hop"); + expect(comboFailureCooldownScope(422, message)).toBe("none"); + } + }); + + test("a structured rate-limit or overload code beside moderation prose keeps its own decision", () => { + for (const code of ["rate_limit_exceeded", "slow_down", "server_is_overloaded"]) { + const message = JSON.stringify({ error: { code, message: "The input was refused by moderation." } }); + expect(isProviderModerationRefusal(400, message, code)).toBe(false); + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureCooldownScope(400, message, { code })).not.toBe("none"); + } + }); +}); From e18e4c7cf60953a218c30ff4b26f62319f9e2f7f Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:39:01 +0900 Subject: [PATCH 16/29] fix(combo): keep response_format, subscription and type verdicts terminal beside moderation Ninth-review follow-ups: - A malformed response_format diagnostic (in the text or the reader's whole-body flag) and subscription-gate wording veto a moderation hop; the classifier files both as generic invalid requests. - Hard-code spellings split camel humps and fold slashes. - Only generic error types (error, invalid_request_error, the gateway SDK wrapper) may stand beside a verdict; any other type stops. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 31 ++++++++++++---- src/lib/errors.ts | 2 +- ...uter-combo-failover-classification.test.ts | 36 +++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 2a6568614af..ce06471e424 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { CYBER_POLICY_FALLBACK_MESSAGE, TERMINAL_REFUSAL_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; +import { CYBER_POLICY_FALLBACK_MESSAGE, TERMINAL_REFUSAL_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isSubscriptionGateMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; import { isHardPolicyOrOriginRefusal } from "../lib/hard-refusal"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; @@ -1028,6 +1028,8 @@ const MODERATION_EVIDENCE_LIMIT = 500; /** Structured codes that can stand beside a moderation verdict without being evidence of their own. */ const MODERATION_COMPATIBLE_CODES = new Set(["moderation_refusal", "content_filter", "invalid_request_error", "400", "422"]); +/** Error types that carry no verdict of their own (the gateway SDK wrapper type included). */ +const MODERATION_COMPATIBLE_TYPES = new Set(["error", "invalid_request_error", "ai_apicallerror", "moderation_error", "content_filter"]); /** * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value @@ -1111,8 +1113,12 @@ const HARD_STOP_TOKEN = * Underscores count as separators, so a token nested in a longer identifier still matches. */ function namesHardStopToken(text: string): boolean { - // A raw JSON body spells whitespace as \n, \t or \r; fold those escapes as well. - return HARD_STOP_TOKEN.test(withJsonEscapesDecoded(text).replace(/\\[ntr]/g, "_").replace(/[-.\s]+/g, "_")); + // A raw JSON body spells whitespace as \n, \t or \r; fold those escapes as well, split camel + // humps (contentPolicyViolation) and treat slashes like the other separators. + return HARD_STOP_TOKEN.test(withJsonEscapesDecoded(text) + .replace(/\\[ntr]/g, "_") + .replace(/([a-z0-9])([A-Z])/g, "$1_$2") + .replace(/[-.\s/]+/g, "_")); } /** JSON \\uXXXX escapes decoded, so an escaped nested code cannot slip past {@link HARD_STOP_TOKEN}. */ @@ -1157,7 +1163,11 @@ function moderationCodeAllowsHop(status: number, normalizedCode: string): boolea * failure reader, the stream preflight) passes its whole-body verdict; the structured code of * this decision can still veto it. Without one, the possibly truncated message decides. */ -function providerModerationHop(status: number, message: string, rawCode: string | null | undefined, wholeBody?: boolean): boolean { +function providerModerationHop( + status: number, message: string, rawCode: string | null | undefined, wholeBody?: boolean, malformedResponseFormat?: boolean, +): boolean { + // A malformed response_format diagnostic the reader saw in the whole body stays terminal. + if (malformedResponseFormat === true) return false; if (wholeBody === undefined) return isProviderModerationRefusal(status, message, rawCode); return wholeBody && moderationCodeAllowsHop(status, normalizedFailureCode(rawCode)); } @@ -1181,6 +1191,10 @@ export function isProviderModerationRefusal( || safetyRefusalCodeFromMessage(message) !== undefined || isCyberPolicyMessage(message) || lower.includes(CYBER_POLICY_FALLBACK_MESSAGE.toLowerCase()) || lower.includes(TERMINAL_REFUSAL_FALLBACK_MESSAGE.toLowerCase()) + // Diagnostics dev keeps terminal although the classifier files them as generic invalid + // requests: a malformed response_format schema and subscription-gate wording. + || isMalformedResponseFormatRequest(status, message) + || isSubscriptionGateMessage(lower) // Classified as a 400: the moderation hop covers 422 too, and classifyError maps an untyped // 422 to upstream_error rather than invalid_request_error. || classifyError(400, "upstream_error", message).code !== "invalid_request_error")) { @@ -1259,6 +1273,11 @@ export function isProviderModerationRefusal( const rootCode = normalizedFailureCode(typeof rec.code === "string" ? rec.code : undefined); const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); + // A verdict stated only as a type is evidence too; only the generic error types may stand + // beside a moderation verdict. + for (const t of [rec.type, e.type]) { + if (typeof t === "string" && t.trim() !== "" && !MODERATION_COMPATIBLE_TYPES.has(normalizedFailureCode(t))) return false; + } for (const c of [rootCode, errCode]) { if ( isCyberPolicyCode(c) @@ -1353,7 +1372,7 @@ export function comboFailureCooldownScope( if (PROVIDER_SCOPED_FAILURE_CODES.has(code)) return "provider"; // A moderation verdict is target-local, but only after quota and credential evidence had its // say: a refusing key beside moderation prose still cools its provider. - if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal)) return "none"; + if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal, options?.malformedResponseFormat)) return "none"; // OpenRouter's opaque numeric-code wrapper may be target-specific (for example a gateway // rejecting a model's translated payload). It may advance the combo, but says nothing about // target health and must not quarantine the target. @@ -1609,7 +1628,7 @@ function decideComboFailure( // reaches that list and returns terminal before anything can ask whether the next target // could have served the request (#4903). if (isResponseFormatCapabilityRefusal(status, message, rawCode)) return "hop"; - if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal)) return "hop"; + if (providerModerationHop(status, message, rawCode, options?.providerModerationRefusal, options?.malformedResponseFormat)) return "hop"; if (isAnthropicBillingFailure(status, message, rawCode)) return "hop"; // Kiro's body validator rejects the request for the identity it was routed under. The account // layer rotates logins first; this is the no-alternate case, so the next combo target is the diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 375c233bdc4..161834fe721 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -214,7 +214,7 @@ export function safetyRefusalCodeFromMessage(text: string): string | undefined { return undefined; } -function isSubscriptionGateMessage(text: string): boolean { +export function isSubscriptionGateMessage(text: string): boolean { return ( text.includes("requires a subscription") || text.includes("requires subscription") || diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index aeb9f33e091..d31f52b4535 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1809,3 +1809,39 @@ describe("provider moderation refusal: escaped whitespace, 422, rate-limit codes } }); }); + +describe("provider moderation refusal: diagnostics dev keeps terminal and verdict types", () => { + const verdict = "The input was refused by moderation. Categories: harassment."; + test("a malformed response_format diagnostic beside a moderation verdict stops", async () => { + for (const raw of [ + JSON.stringify({ error: { code: "content_filter", message: verdict, detail: "Invalid schema for response_format 'reply': 'type' is a required property." } }), + verdict + " response_format.json_schema.schema must be a valid JSON schema.", + ]) { + const failure = await consumeComboFailure(new Response(raw, { status: 400 })); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal, malformedResponseFormat: failure.malformedResponseFormat }; + expect(comboFailureDecision(400, failure.classificationText, options)).toBe("stop"); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal, malformedResponseFormat: failure.malformedResponseFormat })).not.toBe("none"); + } + }); + + test("subscription-gate wording beside a moderation verdict stops", () => { + const message = verdict + " This model requires a subscription."; + expect(isProviderModerationRefusal(400, message)).toBe(false); + expect(comboFailureDecision(400, message)).toBe("stop"); + }); + + test("camelCase and slashed spellings of a hard code stop", () => { + for (const hard of ["contentPolicyViolation", "clientClosedRequest", "content/policy/violation", "upstreamNoResponse"]) { + const message = JSON.stringify({ error: { message: verdict, details: { code: hard } } }); + expect({ hard, hop: isProviderModerationRefusal(400, message) }).toEqual({ hard, hop: false }); + } + }); + + test("a verdict stated only as an error type stops", () => { + for (const type of ["previous_response_not_found", "tool_catalog_too_large", "input_admission_refused", "invalid_parameter"]) { + const message = JSON.stringify({ error: { message: verdict, type } }); + expect({ type, hop: isProviderModerationRefusal(400, message) }).toEqual({ type, hop: false }); + expect(comboFailureDecision(400, message)).toBe("stop"); + } + }); +}); From 98c670d94e85ccae0df3c15eb9538b849fe867b5 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:43:41 +0900 Subject: [PATCH 17/29] refactor(combo): grant a moderation hop on structured evidence only Owner decision on #8428 after nine review rounds: a denylist over prose did not converge, so the hop now needs structured evidence. A body hops only when a moderation_refusal or content_filter code states the verdict (the caller's or one in the body), every other code and type anywhere in the body is on a short compatible list, no unrelated param appears off the coded record, and the existing vetoes (root verdict codes, hard tokens in any spelling, safety and terminal copy, malformed response_format, subscription gate, a specific classifyError verdict) see nothing. Prose alone never hops, so the quote tracker, verdict phrases and truncated-envelope recovery are removed. Tests that expected prose-only hops now assert the terminal decision and the coded hop. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- docs-site/src/content/docs/guides/combos.md | 2 +- .../src/content/docs/ja/guides/combos.md | 2 +- .../src/content/docs/ko/guides/combos.md | 2 +- .../src/content/docs/zh-cn/guides/combos.md | 2 +- src/combos/failover.ts | 396 +++++------------- structure/providers-and-adapters.md | 2 +- ...uter-combo-failover-classification.test.ts | 80 ++-- 7 files changed, 158 insertions(+), 328 deletions(-) diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 8ecb2254014..f512988cdb1 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -699,7 +699,7 @@ provider, see [Fallback off](/guides/fallback-off/). | HTTP 410 with an explicit model end-of-life, retired, deprecated, sunset, decommissioned, or no-longer-available signal | Cool that target and hop. Unrelated 410 responses remain terminal. | | Classified authentication, subscription, quota, rate-limit, overload, or upstream-server error | Cool the target and hop, even when the status alone is not sufficient. | | Confirmed context-window overflow before output, including HTTP 400 | Try the next untried eligible target without cooling this target; another model may have a larger window. Exhaustion returns the first confirmed HTTP 400 context-length error; a context refusal sent with another status still advances but is not retained. | -| HTTP 400 or 422 moderation verdict from the provider or a gateway (`moderation_refusal` or `content_filter` code, or "refused/blocked/flagged by moderation" prose) | Try the next target without cooling this one; another provider's filter may accept the prompt. A body that also carries a terminal refusal (cyber or bio policy, `invalid_prompt`, `content_policy_violation`, safety-system wording) still stops. | +| HTTP 400 or 422 moderation verdict from the provider or a gateway (a structured `moderation_refusal` or `content_filter` code; moderation wording without such a code still stops) | Try the next target without cooling this one; another provider's filter may accept the prompt. A body that also carries a terminal refusal (cyber or bio policy, `invalid_prompt`, `content_policy_violation`, safety-system wording) still stops. | | Client cancellation (499), `origin_rejected`, cyber-policy refusal, or other invalid request | Stop and return the error; another target would not make the request valid. | | Exact HTTP 400 saying a model is not supported when using Codex with a ChatGPT account | Try the next declared target before output commitment. The message must be the whole refusal in `detail`, `error.message`, or bare text; competing `detail` and `error` envelopes stay terminal. This does not quarantine the account or cool every model on the provider. | | Structured HTTP 400 rejecting optional `user`, an unsupported reasoning effort, model-scoped image input, or input media the route cannot read (`unsupported_input_modality`) | For an effort refusal, try the next lower published rung on the same target while the send budget allows; if no rung works, hop before output commitment without cooling if the refusal matches a recognized compatibility shape, otherwise stay terminal. Other listed mismatches hop without cooling; see request-local target compatibility below. | diff --git a/docs-site/src/content/docs/ja/guides/combos.md b/docs-site/src/content/docs/ja/guides/combos.md index 9e0b0552089..b86d67b61b4 100644 --- a/docs-site/src/content/docs/ja/guides/combos.md +++ b/docs-site/src/content/docs/ja/guides/combos.md @@ -128,7 +128,7 @@ ocx combo set balanced \ | HTTP 401、403、404、408、429、または任意の 5xx |ターゲットを冷却し、次の適格なターゲットに移動します。 | |モデルのサポート終了、retired、deprecated、sunset、decommissioned、または利用不可を明示する HTTP 410 |そのターゲットをクールダウンし、次へ進みます。無関係な 410 はターミナル エラーのままです。 | |機密認証、サブスクリプション、クォータ、レート制限、過負荷、またはアップストリーム サーバー エラー |ステータスだけでは物足りない場合でもターゲットを冷やしてホップさせましょう。 | -| プロバイダーまたはゲートウェイの HTTP 400 / 422 モデレーション判定（`moderation_refusal` または `content_filter` コード、あるいは "refused/blocked/flagged by moderation" の文言） |このターゲットを冷却せずに次のターゲットを試行します。別のプロバイダーのフィルターはプロンプトを受け入れる可能性があります。同じ本文に終了扱いの拒否（サイバーまたはバイオポリシー、`invalid_prompt`、`content_policy_violation`、安全システムの文言）が含まれる場合は停止します。 | +| プロバイダーまたはゲートウェイの HTTP 400 / 422 モデレーション判定（構造化された `moderation_refusal` または `content_filter` コード。コードのないモデレーションの文言だけでは停止します） |このターゲットを冷却せずに次のターゲットを試行します。別のプロバイダーのフィルターはプロンプトを受け入れる可能性があります。同じ本文に終了扱いの拒否（サイバーまたはバイオポリシー、`invalid_prompt`、`content_policy_violation`、安全システムの文言）が含まれる場合は停止します。 | |クライアントのキャンセル (499)、`origin_rejected`、サイバー ポリシーの拒否、またはその他の無効なリクエスト |停止してエラーを返します。別のターゲットではリクエストは有効になりません。 | | 出力前に確認したコンテキスト超過（HTTP 400 を含む） | クールダウンせず次の未試行の適格な対象へ進み、使い切った場合は最初に確認した HTTP 400 のコンテキストエラーを返します（他のステータスのコンテキスト拒否も次へ進みますが、保持されません）。 | | ChatGPT アカウントで Codex を使う際に、そのモデルをサポートしないことを示す正確な HTTP 400 の拒否 | 出力を確定する前に、次の設定済みターゲットを試します。拒否文全体が `detail`、`error.message`、または本文そのものにある場合に限ります。`detail` と `error` が競合する応答は終了扱いです。この拒否だけでアカウントを再認証待ちにしたり、プロバイダーの全モデルをクールダウンしたりしません。 | diff --git a/docs-site/src/content/docs/ko/guides/combos.md b/docs-site/src/content/docs/ko/guides/combos.md index ebb4e6e2c76..53589443fda 100644 --- a/docs-site/src/content/docs/ko/guides/combos.md +++ b/docs-site/src/content/docs/ko/guides/combos.md @@ -134,7 +134,7 @@ ocx combo set balanced \ | HTTP 401, 403, 404, 408, 429, 또는 모든 5xx | 대상을 쿨다운으로 보내고 다음 적합한 대상으로 넘어갑니다. | | 모델 수명 종료, retired, deprecated, sunset, decommissioned, 또는 더 이상 사용할 수 없다는 신호가 명시된 HTTP 410 | 해당 대상만 쿨다운으로 보내고 다음 대상으로 넘어갑니다. 관련 없는 410은 종결 오류로 유지합니다. | | 인증, 구독, 쿼터, 속도 제한, 과부하, 또는 상위 서버 오류로 분류됨 | 상태 코드만으로는 충분하지 않더라도 대상을 쿨다운으로 보내고 넘어갑니다. | -| 제공자나 게이트웨이의 HTTP 400 또는 422 moderation 판정(`moderation_refusal` 또는 `content_filter` 코드, 또는 "refused/blocked/flagged by moderation" 문구) | 이 대상을 쿨다운하지 않고 다음 대상을 시도합니다. 다른 제공자의 필터는 프롬프트를 받아들일 수 있습니다. 같은 본문에 종결 거부(cyber 또는 bio 정책, `invalid_prompt`, `content_policy_violation`, 안전 시스템 문구)가 함께 있으면 계속 멈춥니다. | +| 제공자나 게이트웨이의 HTTP 400 또는 422 moderation 판정(구조화된 `moderation_refusal` 또는 `content_filter` 코드. 코드 없이 moderation 문구만 있으면 계속 멈춥니다) | 이 대상을 쿨다운하지 않고 다음 대상을 시도합니다. 다른 제공자의 필터는 프롬프트를 받아들일 수 있습니다. 같은 본문에 종결 거부(cyber 또는 bio 정책, `invalid_prompt`, `content_policy_violation`, 안전 시스템 문구)가 함께 있으면 계속 멈춥니다. | | 클라이언트 취소(499), `origin_rejected`, cyber-policy refusal, 또는 기타 invalid request | 멈추고 오류를 반환합니다. 다른 대상을 써도 요청이 유효해지지 않기 때문입니다. | | 출력 전 확인된 컨텍스트 초과 오류(HTTP 400 포함) | 쿨다운 없이 다음 미시도 적격 대상을 시도합니다. 모두 실패하면 처음 확인된 HTTP 400 컨텍스트 오류를 반환합니다(다른 상태 코드의 컨텍스트 거부도 다음 대상으로 넘어가지만 보관되지 않습니다). | | ChatGPT 계정으로 Codex를 사용할 때 해당 모델을 지원하지 않는다는 정확한 HTTP 400 거부 | 출력이 시작되기 전에 다음 설정 대상으로 넘어갑니다. 거부 문장 전체가 `detail`, `error.message` 또는 일반 텍스트에 있어야 하며, `detail`과 `error`가 충돌하면 중단합니다. 이 사유만으로 계정을 재인증 상태로 바꾸거나 공급자의 모든 모델에 쿨다운을 걸지는 않습니다. | diff --git a/docs-site/src/content/docs/zh-cn/guides/combos.md b/docs-site/src/content/docs/zh-cn/guides/combos.md index 05475bcf0db..4b2316f031f 100644 --- a/docs-site/src/content/docs/zh-cn/guides/combos.md +++ b/docs-site/src/content/docs/zh-cn/guides/combos.md @@ -154,7 +154,7 @@ combo 失败分为 **跳转** 失败和 **终止** 失败。 | HTTP 401、403、404、408、429，或任何 5xx | 使该目标进入冷却，并跳转到下一个合格目标。 | | HTTP 410，并明确表明模型已到生命周期终点、retired、deprecated、sunset、decommissioned 或不再可用 | 仅冷却该目标并继续跳转。无关的 410 仍然是终止错误。 | | 被分类为认证、订阅、配额、速率限制、过载或上游服务器错误 | 即使仅凭状态码不足以判断，也会使该目标进入冷却并跳转。 | -| 提供方或网关的 HTTP 400 或 422 审核判定（`moderation_refusal` 或 `content_filter` 代码，或 "refused/blocked/flagged by moderation" 文字） | 不冷却该目标，尝试下一个目标；其他提供方的过滤器可能接受该提示。若同一响应体中还带有终止性拒绝（cyber 或 bio 策略、`invalid_prompt`、`content_policy_violation`、安全系统文字），仍会停止。 | +| 提供方或网关的 HTTP 400 或 422 审核判定（结构化的 `moderation_refusal` 或 `content_filter` 代码；只有审核文字而没有该代码时仍会停止） | 不冷却该目标，尝试下一个目标；其他提供方的过滤器可能接受该提示。若同一响应体中还带有终止性拒绝（cyber 或 bio 策略、`invalid_prompt`、`content_policy_violation`、安全系统文字），仍会停止。 | | 客户端取消（499）、`origin_rejected`、cyber-policy 拒绝、或其他无效请求 | 停止并返回错误；换其他目标也无法让请求变得有效。 | | 输出前确认的上下文超限（包括 HTTP 400） | 不冷却该目标，尝试下一个尚未尝试的合格目标；耗尽后返回第一个确认的 HTTP 400 上下文错误（其他状态码的上下文拒绝也会切换到下一个目标，但不会被保留）。 | | HTTP 400 明确表示使用 ChatGPT 账户运行 Codex 时不支持该模型 | 在输出提交前尝试下一个已配置目标。完整拒绝句必须位于 `detail`、`error.message` 或纯文本中；同时包含 `detail` 与 `error` 的混合封装仍会终止请求。此原因本身不会将账户标为需要重新登录，也不会让供应商的所有模型进入冷却。 | diff --git a/src/combos/failover.ts b/src/combos/failover.ts index ce06471e424..406df2e5745 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -988,180 +988,65 @@ function isResponseFormatCapabilityRefusal( return false; } -function isAllowedModerationParam(param: string): boolean { - const p = param.trim().toLowerCase(); - return ( - p === "prompt" - || p === "input" - || p === "messages" - || p.startsWith("input[") - || p.startsWith("input.") - || p.startsWith("messages[") - || p.startsWith("messages.") - ); -} - -function isReflectedValidationText(text: string): boolean { - const lower = text.toLowerCase(); - return ( - lower.includes("cannot contain") - || lower.includes("quotes '") - || lower.includes("quotes \"") - || /quotes\s+['"]/i.test(text) - || /text cannot contain/i.test(text) - || /is not supported for (?:field|parameter)\b/i.test(text) - || /invalid (?:parameter|value) '.*?'/i.test(text) - ); -} - -/** Opening quote characters and the character that closes each, ASCII and typographic. */ -const QUOTE_PAIRS: Record = { - "\"": "\"", "`": "`", - "\u201c": "\u201d", "\u201e": "\u201c", "\u2018": "\u2019", "\u201a": "\u2018", "\u00ab": "\u00bb", "\u2039": "\u203a", - "\u300c": "\u300d", "\u300e": "\u300f", "\u3010": "\u3011", "\u301d": "\u301e", "\uff02": "\uff02", "\u2033": "\u2033", "\u2036": "\u2036", - // A closing character used as an opener (\u201dvalue\u201d, \u00bbvalue\u00bb) still quotes. - "\u201d": "\u201d", "\u2019": "\u2019", "\u00bb": "\u00bb", "\u203a": "\u203a", "\u300d": "\u300d", "\u300f": "\u300f", -}; - /** The classification text the combo path bounds a body to (src/server/responses/core-errors.ts). */ const MODERATION_EVIDENCE_LIMIT = 500; -/** Structured codes that can stand beside a moderation verdict without being evidence of their own. */ -const MODERATION_COMPATIBLE_CODES = new Set(["moderation_refusal", "content_filter", "invalid_request_error", "400", "422"]); -/** Error types that carry no verdict of their own (the gateway SDK wrapper type included). */ -const MODERATION_COMPATIBLE_TYPES = new Set(["error", "invalid_request_error", "ai_apicallerror", "moderation_error", "content_filter"]); +/** The structured codes that state a provider or gateway moderation verdict. */ +const MODERATION_REFUSAL_CODES = new Set(["moderation_refusal", "content_filter"]); /** - * A moderation phrase echoed inside quotes is a reflected parameter value (Invalid enum value - * 'blocked by moderation'), not a verdict, so verdict evidence must appear in prose. One pass - * blanks every quoted span; an apostrophe inside a word ("don't") is literal text, not a quote. + * Every other code or type a moderation refusal body may carry. Anything outside this list, at any + * depth, is evidence of something else and the body keeps dev's decision: the hop is granted only + * to bodies whose structured fields all agree that this is a moderation refusal. */ -/** An apostrophe between letters ("it's", "it’s") is literal text, never a quote delimiter. */ -function isInWordApostrophe(text: string, i: number): boolean { - const ch = text[i]; - return (ch === "'" || ch === "\u2019") && i > 0 && /[A-Za-z]/.test(text[i - 1]!) && /[A-Za-z]/.test(text[i + 1] ?? ""); -} +const MODERATION_COMPATIBLE_VALUES = new Set([ + ...MODERATION_REFUSAL_CODES, + "invalid_request_error", "error", "ai_apicallerror", "responsibleaipolicyviolation", "400", "422", +]); -function unquotedProse(text: string): string { - let out = ""; - // The text that ends the open quoted span: one character, a doubled delimiter, or an escaped quote. - let closer: string | null = null; - for (let i = 0; i < text.length;) { - const ch = text[i]!; - if (closer !== null) { - if (text.startsWith(closer, i) && !(closer.length === 1 && isInWordApostrophe(text, i))) { - out += " ".repeat(closer.length); - i += closer.length; - closer = null; - continue; - } - // An escaped character inside a span is content, including an escaped quote of its own kind. - if (ch === "\\" && !closer.startsWith("\\")) { - out += " "; - i += 2; - continue; - } - out += " "; - i += 1; - continue; - } - if (ch === "\\") { - // A backslash-escaped quote (a JSON string read raw) opens a span its escaped twin closes. - const escapedClose = QUOTE_PAIRS[text[i + 1] ?? ""] ?? (text[i + 1] === "'" ? "'" : undefined); - if (escapedClose !== undefined) { - closer = "\\" + escapedClose; - out += " "; - i += 2; - continue; - } - out += ch + (text[i + 1] ?? ""); - i += 2; - continue; - } - const close = isInWordApostrophe(text, i) ? undefined : ch === "'" ? "'" : QUOTE_PAIRS[ch]; - if (close !== undefined) { - // A doubled delimiter ("" or '') opens a span the same doubled delimiter closes. - closer = close === ch && text[i + 1] === ch ? ch + ch : close; - out += " ".repeat(closer.length); - i += closer.length; - continue; - } - out += ch; - i += 1; - } - // An unterminated quote leaves its tail blanked, matching the old "inside a quote" verdict. - return out; -} +/** Keys whose values are structured verdicts rather than prose. */ +const MODERATION_VERDICT_KEYS = new Set(["code", "type"]); -const MODERATION_REFUSAL_VERDICT_PHRASES = [ - "refused by moderation", - "blocked by moderation", - "flagged by moderation", -]; +/** A refusal names the prompt or input it refused; any other `param` is a validation error. */ +function isAllowedModerationParam(param: string): boolean { + const p = param.trim().toLowerCase(); + return p === "prompt" || p === "input" || p === "messages" + || p.startsWith("input[") || p.startsWith("input.") || p.startsWith("messages[") || p.startsWith("messages."); +} /** - * Any hard-refusal or non-replayable code anywhere in the text, structured or serialized. A - * moderation verdict never outranks one: the client must not retry these, and a body that - * carries both is not evidence that another target may serve the request. + * Any hard-refusal token in any spelling. A moderation code beside one is not evidence that another + * target may serve the request: the client must not retry these. */ const HARD_STOP_TOKEN = /(? String.fromCharCode(Number.parseInt(hex, 16))); +} + /** - * Whether any hard-refusal token appears in any spelling: JSON escapes decoded, and hyphens, - * dots and whitespace folded to underscores ("content-policy-violation", "client closed request"). - * Underscores count as separators, so a token nested in a longer identifier still matches. + * Whether any hard-refusal token appears in any spelling: JSON escapes decoded (including \n, \t + * and \r), camel humps split, and hyphens, dots, slashes and whitespace folded to underscores. */ function namesHardStopToken(text: string): boolean { - // A raw JSON body spells whitespace as \n, \t or \r; fold those escapes as well, split camel - // humps (contentPolicyViolation) and treat slashes like the other separators. return HARD_STOP_TOKEN.test(withJsonEscapesDecoded(text) .replace(/\\[ntr]/g, "_") .replace(/([a-z0-9])([A-Z])/g, "$1_$2") .replace(/[-.\s/]+/g, "_")); } -/** JSON \\uXXXX escapes decoded, so an escaped nested code cannot slip past {@link HARD_STOP_TOKEN}. */ -function withJsonEscapesDecoded(text: string): string { - return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); -} - -function namesModerationRefusal(text: string): boolean { - if (isReflectedValidationText(text)) return false; - // Only an explicit verdict phrase counts. A validation error can name moderation categories - // ("Invalid metadata value: moderation categories: harassment") without any refusal. - const prose = unquotedProse(text.toLowerCase()); - return MODERATION_REFUSAL_VERDICT_PHRASES.some(phrase => prose.includes(phrase)); -} - -/** - * A provider or gateway moderation filter refusal is target-local in a heterogeneous combo: - * this model or provider's guardrail refused the prompt, but another target may serve it. - * - * It must precede the generic invalid_request_error stop. - */ /** Whether the caller's status and structured code leave room for a moderation hop at all. */ function moderationCodeAllowsHop(status: number, normalizedCode: string): boolean { if (status !== 400 && status !== 422) return false; - // Any other structured code (rate limit, quota, overload, model scope...) is its own evidence - // with its own cooldown; moderation prose beside it does not turn it into a free hop. - if (normalizedCode !== "" && !MODERATION_COMPATIBLE_CODES.has(normalizedCode)) return false; - return !( - isCyberPolicyCode(normalizedCode) - || isTerminalRefusalCode(normalizedCode) - || isNonReplayableUpstreamCode(normalizedCode) - // The envelope-root verdict codes (origin, content policy, context length, cancellation) stop - // on their own; a moderation phrase beside one cannot hop. A passed invalid_request_error is - // usually the error type standing in for an absent code (policy-fallback reads it that way), - // so it is not a verdict here; as a parsed envelope root below it still is. - || (normalizedCode !== "invalid_request_error" && rootCodeStatesVerdict(normalizedCode)) - ); + return normalizedCode === "" || MODERATION_COMPATIBLE_VALUES.has(normalizedCode); } /** * The moderation hop for one decision. A caller that read the complete bounded body (the combo - * failure reader, the stream preflight) passes its whole-body verdict; the structured code of - * this decision can still veto it. Without one, the possibly truncated message decides. + * failure reader, the stream preflight, the policy path) passes its whole-body verdict; the + * structured code of this decision can still veto it. Without one, the message decides. */ function providerModerationHop( status: number, message: string, rawCode: string | null | undefined, wholeBody?: boolean, malformedResponseFormat?: boolean, @@ -1172,6 +1057,66 @@ function providerModerationHop( return wholeBody && moderationCodeAllowsHop(status, normalizedFailureCode(rawCode)); } +/** Strip this proxy's wrapper prefixes and a single-line SSE data field; undefined unless JSON. */ +function moderationEnvelope(message: string): unknown { + let text = message.trim(); + for (let depth = 0; depth < 3; depth += 1) { + const prefix = ["Provider error 400: ", "Provider error 422: ", "Provider error: ", "Gateway error 400: ", "Gateway error 422: ", "Gateway error: "] + .find(candidate => text.startsWith(candidate)); + if (prefix) text = text.slice(prefix.length).trim(); + if (text.startsWith("data:") && !text.includes("\n")) text = text.slice("data:".length).trim(); + } + try { return JSON.parse(text); } catch { return undefined; } +} + +/** + * Walk every structured verdict field (bounded). Returns whether a moderation code was seen, or + * null when any field states something else, a field holds a non-string value we cannot vouch + * for, or the body is deeper or wider than a refusal envelope. + */ +function moderationVerdictFields(payload: unknown): boolean | null { + let seen = false; + let nodes = 0; + const visit = (value: unknown, depth: number): boolean => { + if (++nodes > 512 || depth > 8) return false; + if (Array.isArray(value)) return value.every(item => visit(item, depth + 1)); + if (!value || typeof value !== "object") return true; + const record = value as Record; + // A record that states the moderation code itself is the verdict; its param names what the + // filter inspected. A param on any other record marks an unrelated validation error. + const recordStatesModeration = typeof record.code === "string" && MODERATION_REFUSAL_CODES.has(normalizedFailureCode(record.code)); + for (const [key, field] of Object.entries(record)) { + if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { + return false; + } + if (MODERATION_VERDICT_KEYS.has(key.toLowerCase())) { + if (field === null || field === undefined) continue; + const normalized = normalizedFailureCode(typeof field === "number" || typeof field === "string" ? field : null); + if (normalized === "" && typeof field !== "string") return false; + if (normalized === "") continue; + if (!MODERATION_COMPATIBLE_VALUES.has(normalized)) return false; + if (MODERATION_REFUSAL_CODES.has(normalized)) seen = true; + continue; + } + if (!visit(field, depth + 1)) return false; + } + return true; + }; + return visit(payload, 0) ? seen : null; +} + +/** + * A provider or gateway moderation refusal on HTTP 400/422 is target-local in a heterogeneous + * combo: this provider's filter refused the prompt, and another target may serve it. + * + * Granted narrowly, on structured evidence only. The verdict must be a structured + * `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other + * code or type anywhere in the body must be compatible with it, and no hard-refusal token, safety or + * terminal copy, malformed response_format diagnostic, subscription gate, or anything the shared + * classifier files more specifically than a generic invalid request may appear. Prose alone never + * hops: wording can be echoed back, and a body that carries both prose and another signal is not + * evidence that the next target may serve the request. + */ export function isProviderModerationRefusal( status: number, message: string, @@ -1181,158 +1126,37 @@ export function isProviderModerationRefusal( ): boolean { const normalizedCode = normalizedFailureCode(code); if (!moderationCodeAllowsHop(status, normalizedCode)) return false; - // Hard refusals anywhere in the body outrank a moderation verdict, including nested codes, - // serialized wrappers and safety copy. The terminal refusal path owns those bodies. - // Beyond the explicit refusal checks, the shared classifier must see nothing more specific than a - // generic invalid request: a body it maps to quota, cancellation or any other verdict is not a - // target-local moderation refusal, whatever prose sits beside it. + const callerStatesModeration = MODERATION_REFUSAL_CODES.has(normalizedCode); + if (!message) return callerStatesModeration; + // Hop only on evidence seen whole: the combo path classifies at most the first + // MODERATION_EVIDENCE_LIMIT characters of a body, and the body readers pass `complete` instead. + if (!complete && message.length >= MODERATION_EVIDENCE_LIMIT) return false; + if (message.length > 32_768) return false; const lower = message.toLowerCase(); - if (message && (namesHardStopToken(message) || isHardPolicyOrOriginRefusal(message) + if (namesHardStopToken(message) || isHardPolicyOrOriginRefusal(message) || safetyRefusalCodeFromMessage(message) !== undefined || isCyberPolicyMessage(message) || lower.includes(CYBER_POLICY_FALLBACK_MESSAGE.toLowerCase()) || lower.includes(TERMINAL_REFUSAL_FALLBACK_MESSAGE.toLowerCase()) - // Diagnostics dev keeps terminal although the classifier files them as generic invalid - // requests: a malformed response_format schema and subscription-gate wording. || isMalformedResponseFormatRequest(status, message) || isSubscriptionGateMessage(lower) - // Classified as a 400: the moderation hop covers 422 too, and classifyError maps an untyped - // 422 to upstream_error rather than invalid_request_error. - || classifyError(400, "upstream_error", message).code !== "invalid_request_error")) { + // Classified as a 400: classifyError maps an untyped 422 to upstream_error. + || classifyError(400, "upstream_error", message).code !== "invalid_request_error") { return false; } - if (!message) { - return normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; + const envelope = moderationEnvelope(message); + if (envelope === undefined) { + // Plain text carries no structured fields; only the caller's code can state the verdict, and a + // body that looks like JSON but does not parse (cut or malformed) never hops. + return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message); } - // Hop only on evidence seen whole. The combo path classifies at most the first - // MODERATION_EVIDENCE_LIMIT characters of a body (normalizeUpstreamErrorText), so a text that - // long may be cut, and a hard refusal after the cut would be invisible to the gate above. The - // body readers pass the whole body with `complete` instead. - if (!complete && message.length >= MODERATION_EVIDENCE_LIMIT) return false; - if (message.length > 32_768) return false; - - let text = message.trim(); - let wasParsed = false; - let moderationCodeSeen = normalizedCode === "moderation_refusal" || normalizedCode === "content_filter"; - - for (let depth = 0; depth < 3; depth += 1) { - // The same wrapper prefixes the hard-refusal parser strips (src/lib/hard-refusal.ts). - const prefix = ["Provider error 400: ", "Provider error 422: ", "Provider error: ", "Gateway error 400: ", "Gateway error: "] - .find(candidate => text.startsWith(candidate)); - if (prefix) text = text.slice(prefix.length).trim(); - if (text.startsWith("data:") && !text.includes("\n")) { - text = text.slice("data:".length).trim(); - } - let payload: unknown; - try { - payload = JSON.parse(text); - wasParsed = true; - } catch { - if (text.startsWith("{") || text.startsWith("[") || wasParsed) { - if ( - isCyberPolicyMessage(text) - || /["']?(?:upstream_no_response|upstream_closed_before_response|upstream_reset_replay_refused|cyber_policy|origin_rejected)["']?/i.test(text) - || isNonReplayableUpstreamCode(normalizedCode) - || normalizedCode === "origin_rejected" - ) { - return false; - } - // Match authoritative error/response envelope containing message and optionally param. - // A dedicated "error": { ... } or "response": { ... } takes precedence over root message. - const envelopeMatch = /"(?:error|response)"\s*:\s*\{([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text) - ?? (!/"(?:error|response)"\s*:\s*\{/.test(text) ? /^\{\s*([^{}]*?"message"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"[^{}]*)/.exec(text) : null); - if (envelopeMatch) { - const envelopeContent = envelopeMatch[1]; - const rawMessage = envelopeMatch[2]; - const paramMatch = /"param"\s*:\s*"([^"\\]*(?:\\.[^"\\]*)*)"/.exec(envelopeContent); - if (paramMatch) { - try { - const extractedParam = JSON.parse(`"${paramMatch[1]}"`) as string; - if (!isAllowedModerationParam(extractedParam)) return false; - } catch { - return false; - } - } - try { - const extractedMsg = JSON.parse(`"${rawMessage}"`) as string; - if (namesModerationRefusal(extractedMsg)) { - return true; - } - } catch { - return false; - } - } - return false; - } - break; - } - if (!payload || typeof payload !== "object" || Array.isArray(payload)) return false; - const rec = payload as Record; - const error = rec.error ?? rec; - if (!error || typeof error !== "object" || Array.isArray(error)) return false; - const e = error as Record; - - const rootCode = normalizedFailureCode(typeof rec.code === "string" ? rec.code : undefined); - const errCode = normalizedFailureCode(typeof e.code === "string" ? e.code : undefined); - // A verdict stated only as a type is evidence too; only the generic error types may stand - // beside a moderation verdict. - for (const t of [rec.type, e.type]) { - if (typeof t === "string" && t.trim() !== "" && !MODERATION_COMPATIBLE_TYPES.has(normalizedFailureCode(t))) return false; - } - for (const c of [rootCode, errCode]) { - if ( - isCyberPolicyCode(c) - || isTerminalRefusalCode(c) - || isNonReplayableUpstreamCode(c) - || c === "origin_rejected" - || c === "content_policy_violation" - || (c !== "" && !MODERATION_COMPATIBLE_CODES.has(c)) - ) { - return false; - } - if (c === "moderation_refusal" || c === "content_filter") { - moderationCodeSeen = true; - } - } - // A root verdict code stops on its own even when the caller passed only the nested code - // (the policy-fallback path reads error.code alone), so read the parsed root here too. - if (rootCode !== "" && rootCodeStatesVerdict(rootCode)) return false; - - const errorObjectModerationCode = errCode === "moderation_refusal" || errCode === "content_filter"; - if (typeof e.param === "string" && e.param) { - if (!isAllowedModerationParam(e.param) && !errorObjectModerationCode) { - return false; - } - } - - if (typeof e.message === "string") { - const msgTrimmed = e.message.trim(); - if (msgTrimmed.startsWith("Provider error 400: ") || msgTrimmed.startsWith("Provider error 422: ") || msgTrimmed.startsWith("{")) { - text = msgTrimmed; - continue; - } - } - - if (moderationCodeSeen) return true; - - const candidateTexts: string[] = []; - if (typeof e.message === "string") candidateTexts.push(e.message); - if (typeof e.error === "string") candidateTexts.push(e.error); - if (typeof rec.message === "string") candidateTexts.push(rec.message); - if (e.param && typeof e.param === "object" && !Array.isArray(e.param)) { - const p = e.param as Record; - if (typeof p.message === "string") candidateTexts.push(p.message); - if (typeof p.error === "string") candidateTexts.push(p.error); - } - - for (const msg of candidateTexts) { - if (namesModerationRefusal(msg)) return true; - } - return false; + // An envelope root that states a verdict code of its own (invalid_request_error, origin, + // context length, cancellation...) keeps it, as everywhere else in this file. + if (envelope && typeof envelope === "object" && !Array.isArray(envelope)) { + const rootCode = (envelope as Record).code; + if (typeof rootCode === "string" && rootCodeStatesVerdict(rootCode)) return false; } - - if (wasParsed || text.startsWith("{") || text.startsWith("[") || text.startsWith("data:")) return false; - if (moderationCodeSeen) return true; - return namesModerationRefusal(text); + const seen = moderationVerdictFields(envelope); + return seen !== null && (seen || callerStatesModeration); } /** Return the narrowest cooldown scope supported by this target's failure evidence. */ diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 8705be7ed2f..516cb59dbf8 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`: a `moderation_refusal` or `content_filter` code, or an unquoted verdict phrase such as "refused by moderation"): that provider's filter is target-local, so the prompt goes to the next combo target. It never outranks a stop: any terminal refusal or root verdict code, non-replayable code, hard-refusal token in any spelling (JSON-escaped, or with hyphens, dots or spaces for underscores), safety or terminal-refusal copy, or any text the shared `classifyError` maps to something more specific than a generic invalid request (quota, cancellation) anywhere in the body keeps the terminal decision and its cooldown. Content-management-policy prose counts only beside a `content_filter` code. The verdict needs the whole body: `consumeComboFailure` and the stream preflight judge the complete bounded body and pass `providerModerationRefusal`, and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure` and the stream preflight judge the complete bounded body and pass `providerModerationRefusal`, and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index d31f52b4535..d5296758959 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1266,19 +1266,22 @@ describe("provider moderation refusal", () => { }, }; - test("hops without cooling on observed gateway moderation refusal", () => { + test("the observed gateway refusal hops only with a structured moderation code", () => { + // Owner decision on #8428: prose alone never hops. The observed body carries no code, so it + // keeps dev's stop; the same body with a structured moderation code hops without cooling. const rawBody = JSON.stringify(moderationPayload); - expect(isProviderModerationRefusal(400, rawBody)).toBe(true); - for (const message of [rawBody, `Provider error 400: ${rawBody}`, `data: ${rawBody}`]) { + expect(isProviderModerationRefusal(400, rawBody)).toBe(false); + expect(comboFailureDecision(400, rawBody)).toBe("stop"); + const coded = JSON.stringify({ ...moderationPayload, error: { ...moderationPayload.error, code: "moderation_refusal" } }); + for (const message of [coded, `Provider error 400: ${coded}`, `data: ${coded}`]) { // The body readers judge the whole body; a wrapped copy is longer than the cut classification text. const providerModerationRefusal = isProviderModerationRefusal(400, message, undefined, true); expect(comboFailureDecision(400, message, { providerModerationRefusal })).toBe("hop"); expect(comboFailureCooldownScope(400, message, { providerModerationRefusal })).toBe("none"); } - expect(comboFailureDecision(400, rawBody)).toBe("hop"); }); - test("hops on various provider moderation phrases", () => { + test("moderation phrases alone stay terminal; the structured code is what hops", () => { const phrases = [ "The input was refused by moderation. Categories: harassment.", "The input was refused by moderation.", @@ -1289,9 +1292,11 @@ describe("provider moderation refusal", () => { ]; for (const phrase of phrases) { const body = JSON.stringify({ error: { message: phrase } }); - expect(isProviderModerationRefusal(400, body)).toBe(true); - expect(comboFailureDecision(400, body)).toBe("hop"); - expect(comboFailureCooldownScope(400, body)).toBe("none"); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + const coded = JSON.stringify({ error: { code: "moderation_refusal", message: phrase } }); + expect(comboFailureDecision(400, coded)).toBe("hop"); + expect(comboFailureCooldownScope(400, coded)).toBe("none"); } }); @@ -1409,24 +1414,24 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, unrelatedField)).toBe("stop"); }); - test("direct moderation verdict after validation prefix hops", () => { + test("a moderation verdict after a validation prefix stays terminal without a code", () => { const directValidation = "Validation error: the prompt was blocked by moderation"; - expect(isProviderModerationRefusal(400, directValidation)).toBe(true); - expect(comboFailureDecision(400, directValidation)).toBe("hop"); - expect(comboFailureCooldownScope(400, directValidation)).toBe("none"); + expect(isProviderModerationRefusal(400, directValidation)).toBe(false); + expect(comboFailureDecision(400, directValidation)).toBe("stop"); const body = JSON.stringify({ error: { message: directValidation } }); - expect(isProviderModerationRefusal(400, body)).toBe(true); - expect(comboFailureDecision(400, body)).toBe("hop"); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(isProviderModerationRefusal(400, directValidation, "content_filter")).toBe(true); }); - test("accepts indexed content parameter paths in moderation refusals", () => { + test("accepts indexed content parameter paths in coded moderation refusals", () => { for (const param of ["input[0].content", "messages[0].content", "messages[1].content[0].text", "input.prompt"]) { const body = JSON.stringify({ error: { message: "The input was refused by moderation. Categories: harassment.", param, }, + code: "moderation_refusal", }); expect(isProviderModerationRefusal(400, body)).toBe(true); expect(comboFailureDecision(400, body)).toBe("hop"); @@ -1471,18 +1476,19 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, moderationWrapperHardCode)).toBe("stop"); }); - test("truncated JSON envelope over 500 characters with intact moderation message hops", async () => { + test("a coded refusal over 500 characters hops on the whole body, never on the cut text", async () => { const padding = "x".repeat(600); const raw = JSON.stringify({ error: { + code: "content_filter", message: "The input was refused by moderation. Categories: harassment.", padding, }, }); - // Truncate after the message field (at 200 characters) + // A cut envelope does not parse, so it carries no structured verdict of its own. const truncated = raw.slice(0, 200); - expect(isProviderModerationRefusal(400, truncated)).toBe(true); - expect(comboFailureDecision(400, truncated)).toBe("hop"); + expect(isProviderModerationRefusal(400, truncated)).toBe(false); + expect(comboFailureDecision(400, truncated)).toBe("stop"); // Pass through consumeComboFailure where production truncation occurs const response = new Response(raw, { status: 400 }); @@ -1543,19 +1549,19 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(400, deepConcealedHardCode)).toBe("stop"); }); - test("unrelated param preceding error envelope in truncated response does not block refusal", async () => { + test("a param on a record other than the coded verdict keeps the body terminal", async () => { const raw = JSON.stringify({ request_info: { param: "unrelated_header" }, error: { + code: "content_filter", message: "The input was refused by moderation. Categories: harassment.", padding: "y".repeat(600), }, }); const response = new Response(raw, { status: 400 }); const failure = await consumeComboFailure(response); - expect(failure.providerModerationRefusal).toBe(true); - expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("hop"); - expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); + expect(failure.providerModerationRefusal).toBe(false); + expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("stop"); }); test("explicit content_filter code triggers failover even when param names other input field", () => { @@ -1679,11 +1685,14 @@ describe("provider moderation refusal judged on the whole body", () => { } }); - test("an error type passed as the code does not veto the canonical moderation refusal", () => { - // The policy-fallback path reads error.type when error.code is null. - const message = JSON.stringify({ error: { message: "The input was refused by moderation. Categories: harassment.", type: "invalid_request_error", param: null, code: null } }); - expect(comboFailureDecision(400, message, { code: "invalid_request_error" })).toBe("hop"); - expect(comboFailureDecision(400, message)).toBe("hop"); + test("an error type passed as the code does not veto a coded moderation refusal", () => { + // The policy-fallback path reads error.type when error.code is null; the canonical uncoded + // OpenAI refusal is prose only and stays terminal. + const uncoded = JSON.stringify({ error: { message: "The input was refused by moderation. Categories: harassment.", type: "invalid_request_error", param: null, code: null } }); + expect(comboFailureDecision(400, uncoded, { code: "invalid_request_error" })).toBe("stop"); + const coded = JSON.stringify({ error: { message: "The input was refused by moderation.", type: "invalid_request_error", param: "input", code: "moderation_refusal" } }); + expect(comboFailureDecision(400, coded, { code: "invalid_request_error" })).toBe("hop"); + expect(comboFailureDecision(400, coded)).toBe("hop"); }); }); @@ -1789,15 +1798,12 @@ describe("provider moderation refusal: escaped whitespace, 422, rate-limit codes } }); - test("a genuine 422 moderation refusal hops without cooling", () => { - for (const body of [ - { error: { code: "content_filter", message: "The input was refused by moderation." } }, - { error: { message: "The prompt was blocked by moderation. Categories: violence" } }, - ]) { - const message = JSON.stringify(body); - expect(comboFailureDecision(422, message)).toBe("hop"); - expect(comboFailureCooldownScope(422, message)).toBe("none"); - } + test("a coded 422 moderation refusal hops without cooling; prose alone does not", () => { + const coded = JSON.stringify({ error: { code: "content_filter", message: "The input was refused by moderation." } }); + expect(comboFailureDecision(422, coded)).toBe("hop"); + expect(comboFailureCooldownScope(422, coded)).toBe("none"); + const prose = JSON.stringify({ error: { message: "The prompt was blocked by moderation. Categories: violence" } }); + expect(comboFailureDecision(422, prose)).toBe("stop"); }); test("a structured rate-limit or overload code beside moderation prose keeps its own decision", () => { From 6e69414612b0fdba2eea773f6aaa87364d6cb19b Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:12:45 +0900 Subject: [PATCH 18/29] fix(combo): carry the whole-event moderation verdict past rebuilt preflight envelopes Final-review follow-ups on the structured-only design: the responses stream preflight and the Messages native preflight judged a moderation refusal on the whole SSE event, then rebuilt a slim error envelope that dropped the root code or innererror, so the combo loop reclassified it as a plain stop (or a 5xx with target cooldown). When the whole event states a moderation verdict, the rebuilt response now carries the event as combo failure evidence, and the native preflight treats that verdict as the request-level 400 it is. Transport parity tests cover both paths. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/server/messages-native-preflight.ts | 20 +++++++--- .../responses/combo-stream-preflight.ts | 8 +++- .../combo-failure-transport-parity.test.ts | 40 +++++++++++++++++++ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/server/messages-native-preflight.ts b/src/server/messages-native-preflight.ts index 42de1326f30..40492fa0e5c 100644 --- a/src/server/messages-native-preflight.ts +++ b/src/server/messages-native-preflight.ts @@ -1,5 +1,6 @@ import { comboFailureDecision } from "../combos"; import { isProviderModerationRefusal } from "../combos/failover"; +import { attachComboFailureEvidence } from "../lib/client-error-body"; import { httpStatusFromTerminalError } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; import { sseFieldValue } from "../lib/sse-decoder"; @@ -71,6 +72,8 @@ export async function preflightNativeMessagesResponse( let terminal = false; let failure: Rec | undefined; let failureStatus = 502; + /** The whole error event, kept when a moderation verdict decided the hop. */ + let failureEvidence: string | undefined; let usage: Rec = {}; const inspector = new BoundedSseFrameBuffer(); const decoder = new TextDecoder(); @@ -94,17 +97,22 @@ export async function preflightNativeMessagesResponse( // Protocol liveness is not output. } else if (payload.type === "error" && isRec(payload.error)) { const error = payload.error; - const status = httpStatusFromTerminalError({ type: typeof error.type === "string" ? error.type : undefined, + const wholeEvent = JSON.stringify(payload); + // A structured moderation verdict on the whole event is a request-level refusal, whatever + // status the terminal mapping infers for its code; judge it like the HTTP reader does. + const moderation = isProviderModerationRefusal(400, wholeEvent, undefined, true); + const mapped = httpStatusFromTerminalError({ type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined, message: typeof error.message === "string" ? error.message : undefined }) ?? 502; + const status = moderation && mapped !== 400 && mapped !== 422 ? 400 : mapped; if (logCtx.terminalSource !== "synthetic" && logCtx.errorCode !== UNDECLARED_TOOL_CALL_ERROR_CODE && comboFailureDecision(status, typeof error.message === "string" ? error.message : "", { code: typeof error.code === "string" ? error.code : undefined, rootCode: typeof payload.code === "string" ? payload.code : undefined, // The whole event, as the HTTP and stream readers judge it, so transports cannot disagree. providerModerationRefusal: status === 400 || status === 422 - ? isProviderModerationRefusal(status, JSON.stringify(payload), undefined, true) : undefined, - }) === "hop") { failure = error; failureStatus = status; } + ? isProviderModerationRefusal(status, wholeEvent, undefined, true) : undefined, + }) === "hop") { failure = error; failureStatus = status; failureEvidence = moderation ? wholeEvent : undefined; } else committed = true; } else if (payload.type === "message_stop") terminal = true; else committed = true; @@ -145,11 +153,13 @@ export async function preflightNativeMessagesResponse( release(); const headers = new Headers(response.headers); headers.set("content-type", "application/json"); headers.delete("content-length"); headers.delete("content-encoding"); - return { kind: "failed", response: Response.json({ error: { + const failed = Response.json({ error: { ...(typeof failure.type === "string" ? { type: failure.type } : {}), ...(typeof failure.code === "string" ? { code: failure.code } : {}), message, - } }, { status: failureStatus, headers }) }; + } }, { status: failureStatus, headers }); + // The rebuilt envelope drops the evidence (root code, innererror) the verdict was judged on. + return { kind: "failed", response: failureEvidence === undefined ? failed : attachComboFailureEvidence(failed, failureEvidence) }; } let index = 0; const body = new ReadableStream({ diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index b82fd451f15..b828a1c722e 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -20,7 +20,7 @@ import { carryResponseToolOwnership } from "./response-tool-ownership"; import { createSseInspector } from "../relay"; import { MAX_CLIENT_SSE_FRAME_BYTES } from "../sse-frame-buffer"; import { REQUIRED_TOOL_CONTRACT_CODE } from "../../lib/errors"; -import { isPausedTurn } from "../../lib/client-error-body"; +import { attachComboFailureEvidence, isPausedTurn } from "../../lib/client-error-body"; import { requiredToolReasoningPrelude } from "./combo-required-tool-stream"; const COMBO_STREAM_PREFLIGHT_MAX_BYTES = MAX_CLIENT_SSE_FRAME_BYTES; @@ -532,6 +532,12 @@ export async function preflightComboStreamResponse( && !outputCommitted && retryableTerminalPayload) { await reader.cancel("retrying zero-output combo stream terminal").catch(() => undefined); const failed = failedTerminalResponse(response, retryableTerminalPayload, logCtx); + // The rebuilt envelope keeps only the record; a moderation verdict judged on the whole frame + // (a root code, an innererror) must reach the combo loop's classification intact. + if ((failed.status === 400 || failed.status === 422) + && isProviderModerationRefusal(failed.status, JSON.stringify(retryableTerminalPayload), undefined, true)) { + attachComboFailureEvidence(failed, JSON.stringify(retryableTerminalPayload)); + } return { kind: "failed", response: failed, ...(failed.status === 400 ? { codexModelRefusal: terminalModelRefusal(failed.status, retryableTerminalPayload) } : {}) }; diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index bfdff799a00..7dc7e8823fa 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -2,6 +2,8 @@ import { describe, expect, test } from "bun:test"; import { comboFailureCooldownScope, comboFailureDecision } from "../../src/combos/failover"; import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; import { preflightComboStreamResponse } from "../../src/server/responses/combo-stream-preflight"; +import { preflightNativeMessagesResponse } from "../../src/server/messages-native-preflight"; +import { createTestTranslatorBudget } from "../helpers/translator-budget"; import type { RequestLogContext } from "../../src/server/request-log"; /** @@ -150,3 +152,41 @@ describe("combo failure verdict parity across HTTP and SSE", () => { }); } }); + +describe("a moderation verdict survives the rebuilt preflight envelope", () => { + const loop = async (response: Response) => { + const failure = await consumeComboFailure(response); + const options = { + code: failure.upstreamCode, rootCode: failure.upstreamRootCode, + malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, + }; + return { + decision: comboFailureDecision(failure.response.status, failure.classificationText, options), + scope: comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal }), + }; + }; + + test("a root-coded responses stream refusal hops without cooling, as over HTTP", async () => { + for (const code of ["content_filter", "moderation_refusal"]) { + const event = { type: "error", status: 400, code, error: { type: "invalid_request_error", message: "The input was refused by moderation." } }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(event) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), logCtx()); + expect(result.kind).toBe("failed"); + expect({ code, ...(await loop(result.response)) }).toEqual({ code, decision: "hop", scope: "none" }); + } + }); + + test("an Azure-style native Messages refusal keeps its moderation evidence", async () => { + const event = { type: "error", error: { code: "content_filter", message: "The response was filtered.", + innererror: { code: "ResponsibleAIPolicyViolation", content_filter_result: { hate: { filtered: true, severity: "high" } } } } }; + const prelude = 'event: message_start\ndata: {"type":"message_start","message":{"content":[],"usage":{"input_tokens":5,"output_tokens":0}}}\n\n'; + const result = await preflightNativeMessagesResponse(new Response( + prelude + "event: error\ndata: " + JSON.stringify(event) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), { model: "fixture", provider: "anth" }, createTestTranslatorBudget()); + expect(result.kind).toBe("failed"); + expect(result.response.status).toBe(400); + expect(await loop(result.response)).toEqual({ decision: "hop", scope: "none" }); + }); +}); From 30583a1993de923d9fa7164dc35053e16605df08 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:24:26 +0900 Subject: [PATCH 19/29] fix(combo): veto safety-system wording and keep a framed native status Final-review follow-ups on the structured-only design: - Safety-system wording in its common upstream phrasings (rejected by or as a result of a safety system, for safety reasons, considered high risk) vetoes a coded moderation hop, beyond the three exact copies safetyRefusalCodeFromMessage names. - The Messages native preflight reads a moderation verdict as a 400 only when the event states no status (or 400/422); a frame that states 429 or 503 keeps the record mapping and its target cooldown, as dev did, and carries moderation evidence only on a 400/422. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 10 +++++++++- src/server/messages-native-preflight.ts | 12 ++++++++++-- .../combo-failure-transport-parity.test.ts | 14 ++++++++++++++ .../router-combo-failover-classification.test.ts | 15 +++++++++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 406df2e5745..577e07e5e9b 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1007,6 +1007,13 @@ const MODERATION_COMPATIBLE_VALUES = new Set([ /** Keys whose values are structured verdicts rather than prose. */ const MODERATION_VERDICT_KEYS = new Set(["code", "type"]); +/** + * Safety-system wording in any of its upstream phrasings ("rejected as a result of our safety + * system", "for safety reasons", "considered high risk"). A safety refusal is the upstream's own + * terminal verdict, broader than the exact copies safetyRefusalCodeFromMessage names. + */ +const SAFETY_SYSTEM_WORDING = /\bsafety[\s_-]+(?:systems?|reasons?)\b|\bhigh[\s_-]+risk\b/i; + /** A refusal names the prompt or input it refused; any other `param` is a validation error. */ function isAllowedModerationParam(param: string): boolean { const p = param.trim().toLowerCase(); @@ -1134,7 +1141,8 @@ export function isProviderModerationRefusal( if (message.length > 32_768) return false; const lower = message.toLowerCase(); if (namesHardStopToken(message) || isHardPolicyOrOriginRefusal(message) - || safetyRefusalCodeFromMessage(message) !== undefined || isCyberPolicyMessage(message) + || safetyRefusalCodeFromMessage(message) !== undefined || SAFETY_SYSTEM_WORDING.test(message) + || isCyberPolicyMessage(message) || lower.includes(CYBER_POLICY_FALLBACK_MESSAGE.toLowerCase()) || lower.includes(TERMINAL_REFUSAL_FALLBACK_MESSAGE.toLowerCase()) || isMalformedResponseFormatRequest(status, message) diff --git a/src/server/messages-native-preflight.ts b/src/server/messages-native-preflight.ts index 40492fa0e5c..b4f03fee3ae 100644 --- a/src/server/messages-native-preflight.ts +++ b/src/server/messages-native-preflight.ts @@ -104,7 +104,12 @@ export async function preflightNativeMessagesResponse( const mapped = httpStatusFromTerminalError({ type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined, message: typeof error.message === "string" ? error.message : undefined }) ?? 502; - const status = moderation && mapped !== 400 && mapped !== 422 ? 400 : mapped; + // Only an event that states no status of its own (or states 400/422) is read as that 400: + // a frame that says 429 or 503 keeps the record mapping, as dev did. + const framed = typeof payload.status === "number" ? payload.status + : typeof error.status === "number" ? error.status : undefined; + const status = moderation && mapped !== 400 && mapped !== 422 + && (framed === undefined || framed === 400 || framed === 422) ? 400 : mapped; if (logCtx.terminalSource !== "synthetic" && logCtx.errorCode !== UNDECLARED_TOOL_CALL_ERROR_CODE && comboFailureDecision(status, typeof error.message === "string" ? error.message : "", { code: typeof error.code === "string" ? error.code : undefined, @@ -112,7 +117,10 @@ export async function preflightNativeMessagesResponse( // The whole event, as the HTTP and stream readers judge it, so transports cannot disagree. providerModerationRefusal: status === 400 || status === 422 ? isProviderModerationRefusal(status, wholeEvent, undefined, true) : undefined, - }) === "hop") { failure = error; failureStatus = status; failureEvidence = moderation ? wholeEvent : undefined; } + }) === "hop") { + failure = error; failureStatus = status; + failureEvidence = moderation && (status === 400 || status === 422) ? wholeEvent : undefined; + } else committed = true; } else if (payload.type === "message_stop") terminal = true; else committed = true; diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index 7dc7e8823fa..781be7f122f 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -190,3 +190,17 @@ describe("a moderation verdict survives the rebuilt preflight envelope", () => { expect(await loop(result.response)).toEqual({ decision: "hop", scope: "none" }); }); }); + +test("a native refusal frame that states its own non-400 status keeps the record mapping and its cooldown", async () => { + const prelude = 'event: message_start\ndata: {"type":"message_start","message":{"content":[],"usage":{"input_tokens":5,"output_tokens":0}}}\n\n'; + for (const status of [429, 503]) { + const event = { type: "error", status, error: { code: "content_filter", message: "The input was refused by moderation." } }; + const result = await preflightNativeMessagesResponse(new Response( + prelude + "event: error\ndata: " + JSON.stringify(event) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), { model: "fixture", provider: "anth" }, createTestTranslatorBudget()); + expect(result.kind).toBe("failed"); + expect(result.response.status).not.toBe(400); + const failure = await consumeComboFailure(result.response); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + } +}); diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index d5296758959..17a889174fe 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1851,3 +1851,18 @@ describe("provider moderation refusal: diagnostics dev keeps terminal and verdic } }); }); + +describe("provider moderation refusal: safety-system wording and framed native status", () => { + test("safety-system wording beside a moderation code stops", () => { + for (const text of [ + "Your request was rejected as a result of our safety system. If you believe this is an error, you can check the status of your account.", + "Your request was rejected by the safety system.", + "The request was rejected because it was considered high risk", + ]) { + const message = JSON.stringify({ type: "error", error: { code: "content_filter", message: text } }); + expect({ text, hop: isProviderModerationRefusal(400, message, "content_filter") }).toEqual({ text, hop: false }); + expect(comboFailureDecision(400, message, { code: "content_filter" })).toBe("stop"); + expect(comboFailureCooldownScope(400, message, { code: "content_filter" })).not.toBe("none"); + } + }); +}); From be3c2794c57c38c904a73585ead6192747e2db41 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:23:18 +0900 Subject: [PATCH 20/29] fix(combo): keep vetoes and framed statuses through rebuilt preflight envelopes Final-review follow-ups: - The responses stream preflight attaches the whole frame as combo failure evidence whenever either the frame or its rebuilt slim envelope reads as a moderation verdict, so a veto the envelope dropped (a root verdict, a nested cyber_policy) is judged by the loop instead of turning into a hop. - The Messages native preflight reads the event's own status with the responses preflight's parser (status, status_code, http_status, numeric or string); a framed non-400/422 status is never a moderation refusal, even when the record type maps to 400. - structure/ names all four whole-body judges. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/server/messages-native-preflight.ts | 20 ++++---- .../responses/combo-stream-preflight.ts | 35 ++++++++++--- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 49 +++++++++++++++++++ 4 files changed, 87 insertions(+), 19 deletions(-) diff --git a/src/server/messages-native-preflight.ts b/src/server/messages-native-preflight.ts index b4f03fee3ae..524a609be4c 100644 --- a/src/server/messages-native-preflight.ts +++ b/src/server/messages-native-preflight.ts @@ -1,6 +1,7 @@ import { comboFailureDecision } from "../combos"; import { isProviderModerationRefusal } from "../combos/failover"; import { attachComboFailureEvidence } from "../lib/client-error-body"; +import { explicitErrorEventStatus } from "./responses/combo-stream-preflight"; import { httpStatusFromTerminalError } from "../lib/errors"; import { redactSecretString } from "../lib/redact"; import { sseFieldValue } from "../lib/sse-decoder"; @@ -98,25 +99,24 @@ export async function preflightNativeMessagesResponse( } else if (payload.type === "error" && isRec(payload.error)) { const error = payload.error; const wholeEvent = JSON.stringify(payload); - // A structured moderation verdict on the whole event is a request-level refusal, whatever - // status the terminal mapping infers for its code; judge it like the HTTP reader does. - const moderation = isProviderModerationRefusal(400, wholeEvent, undefined, true); + // A status the event states for itself (any spelling the responses preflight reads) is + // evidence of its own: a framed 429 or 503 is never read as a moderation refusal. + const framed = explicitErrorEventStatus(payload); + const framedAllowsModeration = framed === undefined || framed === 400 || framed === 422; + // Otherwise a structured moderation verdict on the whole event is a request-level refusal, + // whatever status the terminal mapping infers for its code; judge it like the HTTP reader. + const moderation = framedAllowsModeration && isProviderModerationRefusal(400, wholeEvent, undefined, true); const mapped = httpStatusFromTerminalError({ type: typeof error.type === "string" ? error.type : undefined, code: typeof error.code === "string" ? error.code : undefined, message: typeof error.message === "string" ? error.message : undefined }) ?? 502; - // Only an event that states no status of its own (or states 400/422) is read as that 400: - // a frame that says 429 or 503 keeps the record mapping, as dev did. - const framed = typeof payload.status === "number" ? payload.status - : typeof error.status === "number" ? error.status : undefined; - const status = moderation && mapped !== 400 && mapped !== 422 - && (framed === undefined || framed === 400 || framed === 422) ? 400 : mapped; + const status = moderation && mapped !== 400 && mapped !== 422 ? 400 : mapped; if (logCtx.terminalSource !== "synthetic" && logCtx.errorCode !== UNDECLARED_TOOL_CALL_ERROR_CODE && comboFailureDecision(status, typeof error.message === "string" ? error.message : "", { code: typeof error.code === "string" ? error.code : undefined, rootCode: typeof payload.code === "string" ? payload.code : undefined, // The whole event, as the HTTP and stream readers judge it, so transports cannot disagree. providerModerationRefusal: status === 400 || status === 422 - ? isProviderModerationRefusal(status, wholeEvent, undefined, true) : undefined, + ? framedAllowsModeration && isProviderModerationRefusal(status, wholeEvent, undefined, true) : undefined, }) === "hop") { failure = error; failureStatus = status; failureEvidence = moderation && (status === 400 || status === 422) ? wholeEvent : undefined; diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index b828a1c722e..f0046901e11 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -66,15 +66,18 @@ const RETRYABLE_ZERO_OUTPUT_INCOMPLETE_REASONS = new Set([ "upstream_stall_timeout", ]); -function bareErrorStatus(payload: unknown): number | undefined { +/** + * The HTTP status an error event states for itself (`status`, `status_code` or `http_status`, + * numeric or a three-digit string, on the event or its error record), or undefined. + */ +export function explicitErrorEventStatus(payload: unknown): number | undefined { if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined; const event = payload as Record; - if (event.type !== "error") return undefined; const nested = event.error; const error = nested && typeof nested === "object" && !Array.isArray(nested) ? nested as Record : event; - const explicitStatus = [ + return [ event.status, event.status_code, event.http_status, @@ -88,6 +91,17 @@ function bareErrorStatus(payload: unknown): number | undefined { ? Number(value) : undefined) .find(value => value !== undefined && value >= 400 && value <= 599); +} + +function bareErrorStatus(payload: unknown): number | undefined { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return undefined; + const event = payload as Record; + if (event.type !== "error") return undefined; + const nested = event.error; + const error = nested && typeof nested === "object" && !Array.isArray(nested) + ? nested as Record + : event; + const explicitStatus = explicitErrorEventStatus(payload); const rawCode = error.code ?? event.code; const code = typeof rawCode === "string" ? rawCode @@ -532,11 +546,16 @@ export async function preflightComboStreamResponse( && !outputCommitted && retryableTerminalPayload) { await reader.cancel("retrying zero-output combo stream terminal").catch(() => undefined); const failed = failedTerminalResponse(response, retryableTerminalPayload, logCtx); - // The rebuilt envelope keeps only the record; a moderation verdict judged on the whole frame - // (a root code, an innererror) must reach the combo loop's classification intact. - if ((failed.status === 400 || failed.status === 422) - && isProviderModerationRefusal(failed.status, JSON.stringify(retryableTerminalPayload), undefined, true)) { - attachComboFailureEvidence(failed, JSON.stringify(retryableTerminalPayload)); + // The rebuilt envelope keeps only the record. When either the whole frame or that slim + // envelope reads as a moderation verdict, the combo loop must judge the whole frame: a + // verdict needs the evidence the envelope dropped (a root code, an innererror), and a veto + // the envelope dropped (a root verdict, a nested hard code) must not turn into a hop. + if (failed.status === 400 || failed.status === 422) { + const whole = JSON.stringify(retryableTerminalPayload); + if (isProviderModerationRefusal(failed.status, whole, undefined, true) + || isProviderModerationRefusal(failed.status, await failed.clone().text(), undefined, true)) { + attachComboFailureEvidence(failed, whole); + } } return { kind: "failed", response: failed, ...(failed.status === 400 diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 516cb59dbf8..a82e455e81b 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure` and the stream preflight judge the complete bounded body and pass `providerModerationRefusal`, and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index 781be7f122f..2a16854704a 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -204,3 +204,52 @@ test("a native refusal frame that states its own non-400 status keeps the record expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); } }); + +describe("vetoed moderation bodies agree across transports", () => { + const loopOf = async (response: Response) => { + const failure = await consumeComboFailure(response); + const options = { + code: failure.upstreamCode, rootCode: failure.upstreamRootCode, + malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, + }; + return { + decision: comboFailureDecision(failure.response.status, failure.classificationText, options), + scope: comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal }), + }; + }; + + test("a response.failed frame whose whole body vetoes moderation does not hop after the rebuilt envelope", async () => { + let checked = 0; + const error = { code: "content_filter", message: "The input was refused by moderation." }; + for (const frame of [ + { type: "response.failed", response: { status: "failed", error: { ...error, innererror: { code: "cyber_policy" } } } }, + { type: "response.failed", code: "context_length_exceeded", response: { status: "failed", error } }, + { type: "response.failed", response: { status: "failed", error: { ...error, metadata: { code: "cyber_policy" } } } }, + ]) { + const ctx: RequestLogContext = { ...logCtx(), terminalHttpStatus: 400 }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(frame) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), ctx); + if (result.kind !== "failed") continue; + checked += 1; + expect({ frame: JSON.stringify(frame).slice(0, 60), ...(await loopOf(result.response)) }) + .toEqual({ frame: JSON.stringify(frame).slice(0, 60), decision: "stop", scope: "target" }); + } + expect(checked).toBeGreaterThan(0); + }); + + test("a native frame that states 429 in any spelling is never a moderation hop", async () => { + const prelude = 'event: message_start\ndata: {"type":"message_start","message":{"content":[],"usage":{"input_tokens":5,"output_tokens":0}}}\n\n'; + const record = { type: "invalid_request_error", code: "content_filter", message: "Filtered" }; + for (const framed of [{ status: 429 }, { status: "429" }, { status_code: 429 }, { http_status: 429 }]) { + const event = { type: "error", ...framed, error: record }; + const result = await preflightNativeMessagesResponse(new Response( + prelude + "event: error\ndata: " + JSON.stringify(event) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), { model: "fixture", provider: "anth" }, createTestTranslatorBudget()); + if (result.kind !== "failed") continue; + const outcome = await loopOf(result.response); + expect({ framed, scope: outcome.scope }).not.toEqual({ framed, scope: "none" }); + } + }); +}); From 4a51df9b417fa88f0a4f68e1bdcac429bbaa5de0 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:18:17 +0900 Subject: [PATCH 21/29] fix(combo): read a Responses event discriminator as a carrier, not a verdict A response.failed frame names itself in its root type, which the structured-verdict walk read as an incompatible verdict, so a genuine coded refusal hopped over HTTP and bare error frames but stopped over response.failed once the whole frame was attached. The root response.* type is skipped as the carrier label. The verdict size bound now matches the 64 KiB body readers. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 6 +++++- .../combo-failure-transport-parity.test.ts | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 577e07e5e9b..30c11ccf08e 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1098,6 +1098,9 @@ function moderationVerdictFields(payload: unknown): boolean | null { } if (MODERATION_VERDICT_KEYS.has(key.toLowerCase())) { if (field === null || field === undefined) continue; + // A Responses terminal event names itself in its root `type` (`response.failed`): that is + // the transport's label for the carrier, not the upstream's verdict. + if (depth === 0 && key === "type" && typeof field === "string" && /^response\.[a-z_.]+$/.test(field)) continue; const normalized = normalizedFailureCode(typeof field === "number" || typeof field === "string" ? field : null); if (normalized === "" && typeof field !== "string") return false; if (normalized === "") continue; @@ -1138,7 +1141,8 @@ export function isProviderModerationRefusal( // Hop only on evidence seen whole: the combo path classifies at most the first // MODERATION_EVIDENCE_LIMIT characters of a body, and the body readers pass `complete` instead. if (!complete && message.length >= MODERATION_EVIDENCE_LIMIT) return false; - if (message.length > 32_768) return false; + // The same bound as the bounded body readers (64 KiB), so no carrier silently loses a refusal. + if (message.length > 65_536) return false; const lower = message.toLowerCase(); if (namesHardStopToken(message) || isHardPolicyOrOriginRefusal(message) || safetyRefusalCodeFromMessage(message) !== undefined || SAFETY_SYSTEM_WORDING.test(message) diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index 2a16854704a..78f21abfd65 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -253,3 +253,22 @@ describe("vetoed moderation bodies agree across transports", () => { } }); }); + +test("a genuine coded refusal in a response.failed frame hops without cooling, like the bare error frame and HTTP", async () => { + const error = { type: "invalid_request_error", code: "content_filter", message: "The input was refused by moderation.", + innererror: { code: "ResponsibleAIPolicyViolation", content_filter_result: { hate: { filtered: true, severity: "high" } } } }; + const frame = { type: "response.failed", status: "failed", response: { status: "failed", error } }; + const ctx: RequestLogContext = { ...logCtx(), terminalHttpStatus: 400 }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(frame) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), ctx); + expect(result.kind).toBe("failed"); + const failure = await consumeComboFailure(result.response); + const options = { + code: failure.upstreamCode, rootCode: failure.upstreamRootCode, + malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, + }; + expect(comboFailureDecision(failure.response.status, failure.classificationText, options)).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); +}); From ab8ece99faa95d4733bb7003a42f0a06cca5304b Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:04:12 +0900 Subject: [PATCH 22/29] fix(combo): veto stated statuses and safety flags inside the moderation verdict Final-review follow-ups, applied in the shared verdict so every carrier agrees: - A stated status/status_code/http_status other than 400/422 anywhere in the body vetoes a moderation hop, so an HTTP body that says 429 keeps dev's stop and target cooldown like the stream and native readers. - Verdict-like keys beyond code/type (error_code, error_type, reason, kind, category) must be compatible, and safety or policy flags (safety_violation, blocked, policy_violation) veto, except inside Azure's per-category content-filter results. - A status-less coded stream frame reads as the 400 the other readers judge it as, so it keeps the no-cool verdict. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 28 +++++++++-- .../responses/combo-stream-preflight.ts | 5 ++ structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 50 ++++++++++++++++++- 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 30c11ccf08e..e165affc82f 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1005,7 +1005,18 @@ const MODERATION_COMPATIBLE_VALUES = new Set([ ]); /** Keys whose values are structured verdicts rather than prose. */ -const MODERATION_VERDICT_KEYS = new Set(["code", "type"]); +const MODERATION_VERDICT_KEYS = new Set(["code", "type", "error_code", "error_type", "reason", "kind", "category"]); + +/** Keys that state an HTTP status; any stated status other than 400/422 is evidence of its own. */ +const MODERATION_STATUS_KEYS = new Set(["status", "status_code", "statuscode", "http_status"]); + +/** + * Structured safety or policy flags (`safety_violation`, `blocked`, `policy_violation`). A + * flag is the upstream's own safety verdict; only Azure's per-category content-filter results, + * which describe the moderation verdict itself, may carry such fields. + */ +const SAFETY_FLAG_KEY = /safety|violation|block|polic/i; +const MODERATION_RESULT_KEY = /^content_filter_results?$|^prompt_filter_results?$/i; /** * Safety-system wording in any of its upstream phrasings ("rejected as a result of our safety @@ -1084,9 +1095,9 @@ function moderationEnvelope(message: string): unknown { function moderationVerdictFields(payload: unknown): boolean | null { let seen = false; let nodes = 0; - const visit = (value: unknown, depth: number): boolean => { + const visit = (value: unknown, depth: number, insideFilterResult: boolean): boolean => { if (++nodes > 512 || depth > 8) return false; - if (Array.isArray(value)) return value.every(item => visit(item, depth + 1)); + if (Array.isArray(value)) return value.every(item => visit(item, depth + 1, insideFilterResult)); if (!value || typeof value !== "object") return true; const record = value as Record; // A record that states the moderation code itself is the verdict; its param names what the @@ -1096,6 +1107,13 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { return false; } + if (MODERATION_STATUS_KEYS.has(key.toLowerCase())) { + const stated = typeof field === "number" ? field + : typeof field === "string" && /^\d{3}$/.test(field.trim()) ? Number(field) : undefined; + if (stated !== undefined && stated >= 400 && stated !== 400 && stated !== 422) return false; + if (typeof field !== "object" || field === null) continue; + } + if (!insideFilterResult && SAFETY_FLAG_KEY.test(key)) return false; if (MODERATION_VERDICT_KEYS.has(key.toLowerCase())) { if (field === null || field === undefined) continue; // A Responses terminal event names itself in its root `type` (`response.failed`): that is @@ -1108,11 +1126,11 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (MODERATION_REFUSAL_CODES.has(normalized)) seen = true; continue; } - if (!visit(field, depth + 1)) return false; + if (!visit(field, depth + 1, insideFilterResult || MODERATION_RESULT_KEY.test(key))) return false; } return true; }; - return visit(payload, 0) ? seen : null; + return visit(payload, 0, false) ? seen : null; } /** diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index f0046901e11..b1a0d37c89b 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -109,6 +109,11 @@ function bareErrorStatus(payload: unknown): number | undefined { ? String(rawCode) : null; if (explicitStatus === undefined && code === "invalid_request_error") return 400; + // A structured moderation verdict with no stated status is the request-level 400 the HTTP and + // native readers judge it as, not an unknown code's 5xx. + if (explicitStatus === undefined && isProviderModerationRefusal(400, JSON.stringify(event), undefined, true)) { + return 400; + } // Status is the record's own evidence; whether that record speaks for the envelope is the // decision's question, so a root hard stop still reports 400 instead of inventing a 5xx. if (explicitStatus === undefined && isOpenRouterOpaque400Record(envelopeErrorRecord(event))) { diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index a82e455e81b..5662b05f331 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index 78f21abfd65..f079d911b43 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { comboFailureCooldownScope, comboFailureDecision } from "../../src/combos/failover"; +import { comboFailureCooldownScope, comboFailureDecision, isProviderModerationRefusal } from "../../src/combos/failover"; import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; import { preflightComboStreamResponse } from "../../src/server/responses/combo-stream-preflight"; import { preflightNativeMessagesResponse } from "../../src/server/messages-native-preflight"; @@ -272,3 +272,51 @@ test("a genuine coded refusal in a response.failed frame hops without cooling, l expect(comboFailureDecision(failure.response.status, failure.classificationText, options)).toBe("hop"); expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); }); + +describe("stated statuses and structured flags beside a coded moderation verdict", () => { + const loopOf = async (response: Response) => { + const failure = await consumeComboFailure(response); + const options = { + code: failure.upstreamCode, rootCode: failure.upstreamRootCode, + malformedResponseFormat: failure.malformedResponseFormat, openRouterOpaque400: failure.openRouterOpaque400, + providerModerationRefusal: failure.providerModerationRefusal, + }; + return { + decision: comboFailureDecision(failure.response.status, failure.classificationText, options), + scope: comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal }), + }; + }; + const coded = { type: "invalid_request_error", message: "The input was refused by moderation.", code: "content_filter" }; + + test("a body that states a non-400 status of its own is not a moderation refusal over HTTP", async () => { + for (const stated of [{ status: 429 }, { status: 503 }, { status_code: "429" }, { http_status: "429" }]) { + const body = JSON.stringify({ error: { ...coded, ...stated } }); + const outcome = await loopOf(new Response(body, { status: 400 })); + expect({ stated, outcome }).toEqual({ stated, outcome: { decision: "stop", scope: "target" } }); + } + }); + + test("structured safety or reason fields veto the hop", async () => { + for (const extra of [{ safety_violation: true }, { blocked: true, reason: "policy" }, { reason: "rate_limit_exceeded" }]) { + const body = JSON.stringify({ error: { ...coded, ...extra } }); + expect({ extra, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ extra, verdict: false }); + expect((await loopOf(new Response(body, { status: 400 }))).scope).not.toBe("none"); + } + }); + + test("Azure per-category filter results stay compatible", async () => { + const body = JSON.stringify({ error: { code: "content_filter", message: "The response was filtered.", status: 400, + innererror: { code: "ResponsibleAIPolicyViolation", content_filter_result: { hate: { filtered: true, severity: "high" }, jailbreak: { filtered: false, detected: false } } } } }); + expect(await loopOf(new Response(body, { status: 400 }))).toEqual({ decision: "hop", scope: "none" }); + }); + + test("a status-less coded stream frame keeps the no-cool verdict", async () => { + const event = { type: "error", code: "moderation_refusal" }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(event) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), logCtx()); + expect(result.kind).toBe("failed"); + expect(result.response.status).toBe(400); + expect(await loopOf(result.response)).toEqual({ decision: "hop", scope: "none" }); + }); +}); From e8bfa5ad5280c6debff4f28ee756cf269440431f Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:47:45 +0900 Subject: [PATCH 23/29] fix(combo): veto provider-state wording and read status-less response.failed verdicts as 400 Final-review follow-ups: - Billing, quota and account-state wording (credit balance, quota, payment required, insufficient funds, suspended account, rate limit) and the shared quota and Anthropic billing detectors veto a coded moderation hop, so that provider evidence keeps its cooldown. - A structured moderation verdict in a response.failed frame with no stated status is the 400 the HTTP and native readers judge it as, replacing the log tap's synthetic 502, so it hops without cooling on every carrier. Co-authored-by: Omar <13569609+omar391@users.noreply.github.com> --- src/combos/failover.ts | 10 ++++++- .../responses/combo-stream-preflight.ts | 9 +++++- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 30 +++++++++++++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index e165affc82f..29eed415710 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,5 +1,5 @@ import { parseResetCooldownMs } from "../codex/routing"; -import { CYBER_POLICY_FALLBACK_MESSAGE, TERMINAL_REFUSAL_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isSubscriptionGateMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; +import { CYBER_POLICY_FALLBACK_MESSAGE, TERMINAL_REFUSAL_FALLBACK_MESSAGE, classifyError, isCyberPolicyCode, isCyberPolicyMessage, isRateLimitOrQuotaFailureMessage, isSubscriptionGateMessage, isTerminalRefusalCode, safetyRefusalCodeFromMessage } from "../lib/errors"; import { isHardPolicyOrOriginRefusal } from "../lib/hard-refusal"; import { isNonReplayableUpstreamCode } from "../lib/upstream-retry"; import { isAnthropicBillingFailure } from "../providers/anthropic-billing-refusal"; @@ -1025,6 +1025,10 @@ const MODERATION_RESULT_KEY = /^content_filter_results?$|^prompt_filter_results? */ const SAFETY_SYSTEM_WORDING = /\bsafety[\s_-]+(?:systems?|reasons?)\b|\bhigh[\s_-]+risk\b/i; +/** Billing, quota and account-state wording: evidence about the provider, not about the prompt. */ +const PROVIDER_STATE_WORDING = + /\bcredit[\s_-]+balance\b|\bquota\b|\bbilling\b|\bpayment[\s_-]+required\b|\binsufficient[\s_-]+(?:quota|balance|funds|credits?)\b|\baccount[\s_-]+(?:suspended|disabled|deactivated|banned)\b|\brate[\s_-]+limit/i; + /** A refusal names the prompt or input it refused; any other `param` is a validation error. */ function isAllowedModerationParam(param: string): boolean { const p = param.trim().toLowerCase(); @@ -1169,6 +1173,10 @@ export function isProviderModerationRefusal( || lower.includes(TERMINAL_REFUSAL_FALLBACK_MESSAGE.toLowerCase()) || isMalformedResponseFormatRequest(status, message) || isSubscriptionGateMessage(lower) + // Quota, billing and account-state wording is provider evidence with its own cooldown, even + // where the classifier files a 400 carrying it as a generic invalid request. + || isRateLimitOrQuotaFailureMessage(message) || isAnthropicBillingFailure(400, message, undefined) + || PROVIDER_STATE_WORDING.test(message) // Classified as a 400: classifyError maps an untyped 422 to upstream_error. || classifyError(400, "upstream_error", message).code !== "invalid_request_error") { return false; diff --git a/src/server/responses/combo-stream-preflight.ts b/src/server/responses/combo-stream-preflight.ts index b1a0d37c89b..47c6d037ce3 100644 --- a/src/server/responses/combo-stream-preflight.ts +++ b/src/server/responses/combo-stream-preflight.ts @@ -409,7 +409,14 @@ function failedTerminalResponse( ...(usage && typeof usage === "object" && !Array.isArray(usage) ? { usage } : {}), }, }), { - status: logCtx.terminalHttpStatus ?? bareErrorStatus(terminalPayload) ?? 502, + // A structured moderation verdict on any carrier (`error` or `response.failed`) is the + // request-level 400 the HTTP and native readers judge it as. The verdict already vetoes any + // stated status other than 400/422, so this only replaces a status the log tap inferred (the + // synthetic 502 for an unrecognized code). + status: (logCtx.terminalHttpStatus !== 400 && logCtx.terminalHttpStatus !== 422 + && isProviderModerationRefusal(400, JSON.stringify(terminalPayload), undefined, true)) + ? 400 + : logCtx.terminalHttpStatus ?? bareErrorStatus(terminalPayload) ?? 502, headers, }); } diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 5662b05f331..fb0b93dbda0 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota or credential evidence still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index f079d911b43..3ac0fa09d48 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -320,3 +320,33 @@ describe("stated statuses and structured flags beside a coded moderation verdict expect(await loopOf(result.response)).toEqual({ decision: "hop", scope: "none" }); }); }); + +describe("provider evidence and status-less response.failed frames beside a coded moderation verdict", () => { + test("billing, quota and account-state wording beside a moderation code keeps the provider decision", async () => { + for (const text of [ + "Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.", + "quota exceeded", + "insufficient quota for this request", + "Payment required to continue", + "Account suspended", + ]) { + const body = JSON.stringify({ error: { code: "content_filter", message: text } }); + expect({ text, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ text, verdict: false }); + const failure = await consumeComboFailure(new Response(body, { status: 400 })); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + } + }); + + test("a status-less response.failed moderation frame is the same 400 no-cool hop as over HTTP", async () => { + const frame = { type: "response.failed", response: { status: "failed", error: { code: "content_filter", message: "The input was refused by moderation." } } }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(frame) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), logCtx()); + expect(result.kind).toBe("failed"); + expect(result.response.status).toBe(400); + const failure = await consumeComboFailure(result.response); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal }; + expect(comboFailureDecision(400, failure.classificationText, options)).toBe("hop"); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); + }); +}); From ee834aea96935edfdbe649ebb8024d5c1557c225 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:13:45 +0900 Subject: [PATCH 24/29] fix(combo): require moderation prose beside a coded verdict and fold verdict keys A content_filter code beside a parameter error, a missing model or an inactive account hopped with no cooldown, though dev stopped and cooled those bodies. Every prose field beside the code must now describe the moderation refusal. Account, organization and API-key state wording and flags veto the hop, and status and verdict keys are compared folded so httpStatus and errorCode count. --- src/combos/failover.ts | 34 +++++++++++++++---- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 34 +++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 29eed415710..2608432116a 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1005,17 +1005,33 @@ const MODERATION_COMPATIBLE_VALUES = new Set([ ]); /** Keys whose values are structured verdicts rather than prose. */ -const MODERATION_VERDICT_KEYS = new Set(["code", "type", "error_code", "error_type", "reason", "kind", "category"]); +// Keys are compared folded: lowercase with underscores and hyphens removed (errorCode, error_code). +const MODERATION_VERDICT_KEYS = new Set(["code", "type", "errorcode", "errortype", "reason", "kind", "category"]); /** Keys that state an HTTP status; any stated status other than 400/422 is evidence of its own. */ -const MODERATION_STATUS_KEYS = new Set(["status", "status_code", "statuscode", "http_status"]); +const MODERATION_STATUS_KEYS = new Set(["status", "statuscode", "httpstatus", "httpstatuscode"]); + +/** Keys whose string values are the body's prose. */ +const MODERATION_PROSE_KEYS = new Set(["message", "error", "detail", "details", "msg", "description", "errormessage"]); + +/** + * Every prose string beside a coded verdict must itself describe the moderation refusal. A body + * that also says something else (a parameter error, a missing model, an inactive account) carries + * evidence of its own, so the verdict is not the whole story and the body keeps dev's decision. + */ +const MODERATION_PROSE = /moderat|content[\s_-]*(?:filter|management|safety)|\bfiltered\b|responsible[\s_-]*ai/i; + +/** Fold a key for comparison: lowercase, underscores and hyphens removed. */ +function foldKey(key: string): string { + return key.toLowerCase().replace(/[_-]/g, ""); +} /** * Structured safety or policy flags (`safety_violation`, `blocked`, `policy_violation`). A * flag is the upstream's own safety verdict; only Azure's per-category content-filter results, * which describe the moderation verdict itself, may carry such fields. */ -const SAFETY_FLAG_KEY = /safety|violation|block|polic/i; +const SAFETY_FLAG_KEY = /safety|violation|block|polic|account|suspend|disabl|deactivat|banned/i; const MODERATION_RESULT_KEY = /^content_filter_results?$|^prompt_filter_results?$/i; /** @@ -1027,7 +1043,7 @@ const SAFETY_SYSTEM_WORDING = /\bsafety[\s_-]+(?:systems?|reasons?)\b|\bhigh[\s_ /** Billing, quota and account-state wording: evidence about the provider, not about the prompt. */ const PROVIDER_STATE_WORDING = - /\bcredit[\s_-]+balance\b|\bquota\b|\bbilling\b|\bpayment[\s_-]+required\b|\binsufficient[\s_-]+(?:quota|balance|funds|credits?)\b|\baccount[\s_-]+(?:suspended|disabled|deactivated|banned)\b|\brate[\s_-]+limit/i; + /\bcredit[\s_-]+balance\b|\bquota\b|\bbilling\b|\bpayment[\s_-]+required\b|\binsufficient[\s_-]+(?:quota|balance|funds|credits?)\b|\b(?:account|organi[sz]ation|api[\s_-]*key)\b[^.]{0,40}?\b(?:suspended|disabled|deactivated|banned|inactive|not[\s_-]+active)\b|\brate[\s_-]+limit/i; /** A refusal names the prompt or input it refused; any other `param` is a validation error. */ function isAllowedModerationParam(param: string): boolean { @@ -1111,14 +1127,18 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { return false; } - if (MODERATION_STATUS_KEYS.has(key.toLowerCase())) { + if (MODERATION_STATUS_KEYS.has(foldKey(key))) { const stated = typeof field === "number" ? field : typeof field === "string" && /^\d{3}$/.test(field.trim()) ? Number(field) : undefined; if (stated !== undefined && stated >= 400 && stated !== 400 && stated !== 422) return false; if (typeof field !== "object" || field === null) continue; } if (!insideFilterResult && SAFETY_FLAG_KEY.test(key)) return false; - if (MODERATION_VERDICT_KEYS.has(key.toLowerCase())) { + if (MODERATION_PROSE_KEYS.has(foldKey(key)) && typeof field === "string" && field.trim() !== "" + && !MODERATION_PROSE.test(field)) { + return false; + } + if (MODERATION_VERDICT_KEYS.has(foldKey(key))) { if (field === null || field === undefined) continue; // A Responses terminal event names itself in its root `type` (`response.failed`): that is // the transport's label for the carrier, not the upstream's verdict. @@ -1185,7 +1205,7 @@ export function isProviderModerationRefusal( if (envelope === undefined) { // Plain text carries no structured fields; only the caller's code can state the verdict, and a // body that looks like JSON but does not parse (cut or malformed) never hops. - return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message); + return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message) && MODERATION_PROSE.test(message); } // An envelope root that states a verdict code of its own (invalid_request_error, origin, // context length, cancellation...) keeps it, as everywhere else in this file. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index fb0b93dbda0..5f37ce7bc62 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Every prose field beside the code (`message`, `detail`, `description`...) must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index 3ac0fa09d48..ab9ddd18ace 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -350,3 +350,37 @@ describe("provider evidence and status-less response.failed frames beside a code expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); }); }); + +describe("every prose field beside a coded verdict must describe the moderation refusal", () => { + const coded = (message: string, extra: Record = {}) => + JSON.stringify({ error: { type: "invalid_request_error", code: "content_filter", message, ...extra } }); + + test("parameter, model and account prose beside a moderation code keeps dev's decision", async () => { + for (const body of [ + coded("Invalid 'temperature': maximum value is 2."), + coded("The model gpt-9 does not exist"), + coded("This account has been deactivated."), + coded("Your account is disabled."), + coded("This organization has been disabled."), + coded("Your API key is disabled."), + coded("The request was rejected because the account is not active."), + coded("The input was refused by moderation.", { accountSuspended: true }), + coded("The input was refused by moderation.", { metadata: { account_status: "suspended" } }), + coded("The input was refused by moderation.", { httpStatus: 429 }), + ]) { + expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: false }); + const failure = await consumeComboFailure(new Response(body, { status: 400 })); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + } + }); + + test("genuine coded refusals still hop", () => { + for (const body of [ + coded("The input was refused by moderation. Categories: harassment."), + JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry.", innererror: { code: "ResponsibleAIPolicyViolation" } } }), + JSON.stringify({ error: { code: "moderation_refusal", message: "Your request was flagged by moderation." } }), + ]) { + expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: true }); + } + }); +}); From f7fcc51a39dd2701c9b51cdc22b44a72f73ce847 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:37:39 +0900 Subject: [PATCH 25/29] fix(combo): check prose under any key and read only the error record of a response.failed carrier Prose beside a content_filter code under keys other than message (hint, note, metadata.raw) still hopped without cooling, though dev stopped those bodies. Every prose string in the body must now describe the moderation refusal. A real response.failed event carries the whole response object, whose tools and text format have their own type fields; the verdict walk read them as incompatible codes, so a genuine coded refusal on that carrier stopped. The walk now reads only the response object's error record. --- src/combos/failover.ts | 13 +++++++++- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 25 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 2608432116a..604351ba375 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1123,7 +1123,14 @@ function moderationVerdictFields(payload: unknown): boolean | null { // A record that states the moderation code itself is the verdict; its param names what the // filter inspected. A param on any other record marks an unrelated validation error. const recordStatesModeration = typeof record.code === "string" && MODERATION_REFUSAL_CODES.has(normalizedFailureCode(record.code)); + // A Responses terminal event (`response.failed`) carries the whole response object, which echoes + // the caller's instructions, tools and text format; only its error record is the verdict. + const responseCarrier = depth === 0 && typeof record.type === "string" && /^response\.[a-z_.]+$/.test(record.type); for (const [key, field] of Object.entries(record)) { + if (responseCarrier && key === "response" && field && typeof field === "object" && !Array.isArray(field)) { + if (!visit((field as Record).error, depth + 1, insideFilterResult)) return false; + continue; + } if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { return false; } @@ -1134,7 +1141,11 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (typeof field !== "object" || field === null) continue; } if (!insideFilterResult && SAFETY_FLAG_KEY.test(key)) return false; - if (MODERATION_PROSE_KEYS.has(foldKey(key)) && typeof field === "string" && field.trim() !== "" + // Prose beside the verdict, under any key (message, hint, note, metadata.raw...), must itself + // describe the moderation refusal; a bare token (an id, an enum) is not prose. + if (typeof field === "string" && field.trim() !== "" && !insideFilterResult + && key.toLowerCase() !== "param" && !MODERATION_VERDICT_KEYS.has(foldKey(key)) + && (MODERATION_PROSE_KEYS.has(foldKey(key)) || /\s/.test(field.trim())) && !MODERATION_PROSE.test(field)) { return false; } diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 5f37ce7bc62..92ddb54ba38 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier. Every prose field beside the code (`message`, `detail`, `description`...) must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object contributes only its `error` record (it also echoes the caller's instructions, tools and text format). Every prose string in the body, under any key (`message`, `detail`, `hint`, `metadata.raw`...), must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index ab9ddd18ace..f490a5c6e95 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -367,6 +367,9 @@ describe("every prose field beside a coded verdict must describe the moderation coded("The input was refused by moderation.", { accountSuspended: true }), coded("The input was refused by moderation.", { metadata: { account_status: "suspended" } }), coded("The input was refused by moderation.", { httpStatus: 429 }), + coded("The input was refused by moderation.", { hint: "The model gpt-9 does not exist" }), + coded("The input was refused by moderation.", { note: "Invalid 'temperature': maximum value is 2." }), + coded("The input was refused by moderation.", { metadata: { raw: "Your account is past due" } }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: false }); const failure = await consumeComboFailure(new Response(body, { status: 400 })); @@ -379,8 +382,30 @@ describe("every prose field beside a coded verdict must describe the moderation coded("The input was refused by moderation. Categories: harassment."), JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry.", innererror: { code: "ResponsibleAIPolicyViolation" } } }), JSON.stringify({ error: { code: "moderation_refusal", message: "Your request was flagged by moderation." } }), + coded("The input was refused by moderation.", { request_id: "req_abc123" }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: true }); } }); + + test("a response.failed carrier's echoed instructions and tools do not veto its coded refusal", async () => { + const frame = { + type: "response.failed", + response: { + status: "failed", + model: "gpt-test", + instructions: "You are a helpful assistant. Answer briefly.", + tools: [{ type: "function", name: "lookup", description: "Look up an order by its number." }], + error: { code: "content_filter", message: "The input was refused by moderation." }, + }, + }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(frame) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), logCtx()); + expect(result.kind).toBe("failed"); + const failure = await consumeComboFailure(result.response); + const options = { code: failure.upstreamCode, rootCode: failure.upstreamRootCode, providerModerationRefusal: failure.providerModerationRefusal }; + expect(comboFailureDecision(failure.response.status, failure.classificationText, options)).toBe("hop"); + expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); + }); }); From c2a7c88a9a40f8ba6fee872facdf6692905423c2 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:39:58 +0900 Subject: [PATCH 26/29] fix(combo): read a response.failed carrier without its request echoes only The previous commit read only the error record of a response.failed response object, so a stated status, a safety flag or unrelated prose elsewhere on that object no longer vetoed the no-cool hop. The walk now reads the whole response object except the fields that echo the caller's request (instructions, tools, text, reasoning, input, output, metadata...). --- src/combos/failover.ts | 13 +++++++++++-- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 604351ba375..a5a0a9aed51 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1026,6 +1026,12 @@ function foldKey(key: string): string { return key.toLowerCase().replace(/[_-]/g, ""); } +/** Response-object fields that echo the caller's request rather than describe the failure. */ +const RESPONSE_REQUEST_ECHO_KEYS = new Set([ + "instructions", "tools", "tool_choice", "text", "reasoning", "input", "output", "prompt", "include", + "metadata", "user", "safety_identifier", "prompt_cache_key", "previous_response_id", "conversation", +]); + /** * Structured safety or policy flags (`safety_violation`, `blocked`, `policy_violation`). A * flag is the upstream's own safety verdict; only Azure's per-category content-filter results, @@ -1124,11 +1130,14 @@ function moderationVerdictFields(payload: unknown): boolean | null { // filter inspected. A param on any other record marks an unrelated validation error. const recordStatesModeration = typeof record.code === "string" && MODERATION_REFUSAL_CODES.has(normalizedFailureCode(record.code)); // A Responses terminal event (`response.failed`) carries the whole response object, which echoes - // the caller's instructions, tools and text format; only its error record is the verdict. + // the caller's request (instructions, tools, text format...). Those echoes say nothing about the + // verdict; every other field of the response object is still read. const responseCarrier = depth === 0 && typeof record.type === "string" && /^response\.[a-z_.]+$/.test(record.type); for (const [key, field] of Object.entries(record)) { if (responseCarrier && key === "response" && field && typeof field === "object" && !Array.isArray(field)) { - if (!visit((field as Record).error, depth + 1, insideFilterResult)) return false; + const verdictFields = Object.fromEntries(Object.entries(field as Record) + .filter(([responseKey]) => !RESPONSE_REQUEST_ECHO_KEYS.has(responseKey))); + if (!visit(verdictFields, depth + 1, insideFilterResult)) return false; continue; } if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 92ddb54ba38..5398cd81fdd 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object contributes only its `error` record (it also echoes the caller's instructions, tools and text format). Every prose string in the body, under any key (`message`, `detail`, `hint`, `metadata.raw`...), must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`, `metadata`...). Every prose string in the body, under any key (`message`, `detail`, `hint`, `metadata.raw`...), must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index f490a5c6e95..c4998728aab 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -408,4 +408,23 @@ describe("every prose field beside a coded verdict must describe the moderation expect(comboFailureDecision(failure.response.status, failure.classificationText, options)).toBe("hop"); expect(comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("none"); }); + + test("evidence on a response.failed carrier outside its error record still vetoes the no-cool hop", async () => { + const error = { code: "content_filter", message: "The input was refused by moderation." }; + for (const extra of [ + { status_code: 429 }, + { safety_violation: true }, + { note: "The model gpt-9 does not exist" }, + { incomplete_details: { reason: "Your account is past due" } }, + ]) { + const frame = { type: "response.failed", response: { status: "failed", ...extra, error } }; + const result = await preflightComboStreamResponse(new Response( + "data: " + JSON.stringify(frame) + "\n\n", { headers: { "content-type": "text/event-stream" } }, + ), logCtx()); + expect(result.kind).toBe("failed"); + const failure = await consumeComboFailure(result.response); + const scope = comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal }); + expect({ extra, scope }).not.toEqual({ extra, scope: "none" }); + } + }); }); From 8826e2c6522ed4270435007468eb17bdc629ab18 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:12:37 +0900 Subject: [PATCH 27/29] fix(combo): check params on every record, array prose, mixed clauses and response metadata Four bodies that dev stops and cools still hopped without cooling: - a param outside the prompt/input paths on the record that carries the moderation code (param "temperature"); - prose inside array items (details: ["The model gpt-9 does not exist"]); - one prose field that mixes moderation wording with other evidence ("Refused by moderation. Your account is past due."); - evidence inside a response.failed frame's response.metadata. Params are now checked on every record, array strings are prose like any other field, each clause of a prose string must name the refusal or be a known follow-up (categories, change the prompt and retry, documentation), and response.metadata is read. --- src/combos/failover.ts | 39 +++++++++++++++---- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 12 ++++++ ...uter-combo-failover-classification.test.ts | 8 ++-- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index a5a0a9aed51..4a15d67fc6c 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1021,6 +1021,25 @@ const MODERATION_PROSE_KEYS = new Set(["message", "error", "detail", "details", */ const MODERATION_PROSE = /moderat|content[\s_-]*(?:filter|management|safety)|\bfiltered\b|responsible[\s_-]*ai/i; +/** + * Clauses a refusal may carry beside its verdict without adding evidence of its own: the category + * list, the advice to change the prompt and retry, and the pointer to documentation. + */ +const MODERATION_PROSE_FOLLOW_UP = + /^(?:categor(?:y|ies)\b.*|(?:please\s+)?(?:modify|revise|rephrase|change|adjust|edit|update)\s+(?:your|the)\s+(?:prompt|input|request|message|content)s?\b.*|(?:please\s+)?(?:try|retry)(?:\s+again)?\b.*|.*\b(?:learn\s+more|more\s+information|documentation)\b.*)$/i; + +/** + * Whether a prose string only describes the moderation refusal. Each clause (split at sentence + * ends, semicolons, commas and joining words) must name the refusal or be a known follow-up, so a + * string that also reports an account, model, parameter or billing problem keeps dev's decision. + */ +function describesOnlyModeration(text: string): boolean { + const clauses = text.split(/(?<=[.!?;,])\s+|\s+(?:and|but|however|while|although|because)\s+|\n+/i) + .map(clause => clause.trim()).filter(clause => clause !== ""); + return clauses.some(clause => MODERATION_PROSE.test(clause)) + && clauses.every(clause => MODERATION_PROSE.test(clause) || MODERATION_PROSE_FOLLOW_UP.test(clause)); +} + /** Fold a key for comparison: lowercase, underscores and hyphens removed. */ function foldKey(key: string): string { return key.toLowerCase().replace(/[_-]/g, ""); @@ -1029,7 +1048,7 @@ function foldKey(key: string): string { /** Response-object fields that echo the caller's request rather than describe the failure. */ const RESPONSE_REQUEST_ECHO_KEYS = new Set([ "instructions", "tools", "tool_choice", "text", "reasoning", "input", "output", "prompt", "include", - "metadata", "user", "safety_identifier", "prompt_cache_key", "previous_response_id", "conversation", + "user", "safety_identifier", "prompt_cache_key", "previous_response_id", "conversation", ]); /** @@ -1123,12 +1142,14 @@ function moderationVerdictFields(payload: unknown): boolean | null { let nodes = 0; const visit = (value: unknown, depth: number, insideFilterResult: boolean): boolean => { if (++nodes > 512 || depth > 8) return false; - if (Array.isArray(value)) return value.every(item => visit(item, depth + 1, insideFilterResult)); + if (Array.isArray(value)) { + // A string item is prose like any other field (`details: ["..."]`). + return value.every(item => typeof item === "string" + ? insideFilterResult || !/\s/.test(item.trim()) || describesOnlyModeration(item) + : visit(item, depth + 1, insideFilterResult)); + } if (!value || typeof value !== "object") return true; const record = value as Record; - // A record that states the moderation code itself is the verdict; its param names what the - // filter inspected. A param on any other record marks an unrelated validation error. - const recordStatesModeration = typeof record.code === "string" && MODERATION_REFUSAL_CODES.has(normalizedFailureCode(record.code)); // A Responses terminal event (`response.failed`) carries the whole response object, which echoes // the caller's request (instructions, tools, text format...). Those echoes say nothing about the // verdict; every other field of the response object is still read. @@ -1140,7 +1161,9 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (!visit(verdictFields, depth + 1, insideFilterResult)) return false; continue; } - if (key.toLowerCase() === "param" && !recordStatesModeration && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { + // A param names what the filter inspected: the prompt or input. Any other param, on any + // record, marks a validation error. + if (key.toLowerCase() === "param" && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { return false; } if (MODERATION_STATUS_KEYS.has(foldKey(key))) { @@ -1155,7 +1178,7 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (typeof field === "string" && field.trim() !== "" && !insideFilterResult && key.toLowerCase() !== "param" && !MODERATION_VERDICT_KEYS.has(foldKey(key)) && (MODERATION_PROSE_KEYS.has(foldKey(key)) || /\s/.test(field.trim())) - && !MODERATION_PROSE.test(field)) { + && !describesOnlyModeration(field)) { return false; } if (MODERATION_VERDICT_KEYS.has(foldKey(key))) { @@ -1225,7 +1248,7 @@ export function isProviderModerationRefusal( if (envelope === undefined) { // Plain text carries no structured fields; only the caller's code can state the verdict, and a // body that looks like JSON but does not parse (cut or malformed) never hops. - return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message) && MODERATION_PROSE.test(message); + return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message) && describesOnlyModeration(message); } // An envelope root that states a verdict code of its own (invalid_request_error, origin, // context length, cancellation...) keeps it, as everywhere else in this file. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 5398cd81fdd..d33c17a4f6c 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths except on the coded record; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`, `metadata`...). Every prose string in the body, under any key (`message`, `detail`, `hint`, `metadata.raw`...), must itself describe a moderation or content-filter refusal; a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths on any record, the coded one included; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`...; `metadata` is still read). Every prose string in the body, under any key or inside an array (`message`, `detail`, `hint`, `metadata.raw`, `details[]`...), must only describe a moderation or content-filter refusal: each clause names the refusal or is a known follow-up (categories, change the prompt and retry, documentation); a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index c4998728aab..d1751161007 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -370,6 +370,13 @@ describe("every prose field beside a coded verdict must describe the moderation coded("The input was refused by moderation.", { hint: "The model gpt-9 does not exist" }), coded("The input was refused by moderation.", { note: "Invalid 'temperature': maximum value is 2." }), coded("The input was refused by moderation.", { metadata: { raw: "Your account is past due" } }), + coded("The input was refused by moderation.", { param: "temperature" }), + coded("The input was refused by moderation.", { param: "max_tokens" }), + coded("The input was refused by moderation.", { details: ["The model gpt-9 does not exist"] }), + coded("The input was refused by moderation.", { details: ["Your account is past due"] }), + coded("The input was refused by moderation.", { note: "Refused by moderation. Your account is past due." }), + coded("The input was refused by moderation.", { note: "Refused by moderation; your organization is over its spend limit." }), + coded("The input was refused by moderation.", { note: "The model gpt-9 does not exist BUT the input was refused by moderation." }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: false }); const failure = await consumeComboFailure(new Response(body, { status: 400 })); @@ -383,6 +390,8 @@ describe("every prose field beside a coded verdict must describe the moderation JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry.", innererror: { code: "ResponsibleAIPolicyViolation" } } }), JSON.stringify({ error: { code: "moderation_refusal", message: "Your request was flagged by moderation." } }), coded("The input was refused by moderation.", { request_id: "req_abc123" }), + coded("The input was refused by moderation.", { param: "input", details: ["harassment"] }), + JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry. To learn more about our content filtering policies please read our documentation: https://go.microsoft.com/fwlink/?linkid=2198766" } }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: true }); } @@ -416,6 +425,9 @@ describe("every prose field beside a coded verdict must describe the moderation { safety_violation: true }, { note: "The model gpt-9 does not exist" }, { incomplete_details: { reason: "Your account is past due" } }, + { metadata: { raw: "Your account is past due" } }, + { metadata: { httpStatus: 503 } }, + { metadata: { safety_violation: true } }, ]) { const frame = { type: "response.failed", response: { status: "failed", ...extra, error } }; const result = await preflightComboStreamResponse(new Response( diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 17a889174fe..912655ac796 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1564,7 +1564,7 @@ describe("provider moderation refusal", () => { expect(comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).toBe("stop"); }); - test("explicit content_filter code triggers failover even when param names other input field", () => { + test("a param outside the prompt or input keeps the body terminal even on the coded record", () => { const body = JSON.stringify({ error: { code: "content_filter", @@ -1572,9 +1572,9 @@ describe("provider moderation refusal", () => { param: "instructions", }, }); - expect(isProviderModerationRefusal(400, body)).toBe(true); - expect(comboFailureDecision(400, body)).toBe("hop"); - expect(comboFailureCooldownScope(400, body)).toBe("none"); + expect(isProviderModerationRefusal(400, body)).toBe(false); + expect(comboFailureDecision(400, body)).toBe("stop"); + expect(comboFailureCooldownScope(400, body)).not.toBe("none"); }); test("root content_filter code does not exempt nested validation error with disallowed param", () => { From f6210ffb5cc204aed99de7f37cd685e2beb3a6c5 Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:52:42 +0900 Subject: [PATCH 28/29] fix(combo): admit only the known shape of a moderation refusal Checking free text for contradicting evidence kept leaking: separators the clause splitter missed, follow-up clauses carrying account or model evidence, bare state tokens under unknown fields, and a quadratic split on long whitespace runs. A multi-category list was also split at its comma and lost. The verdict walk now admits a closed set of fields (codes, a 400/422 status, a prompt/input param or an SDK error object, identifiers, innererror, Azure's filter results, and a gateway's routing metadata of plain tokens), and prose must match known refusal copy as a whole: one verdict sentence plus optional categories, the advice to change the prompt and retry, and Azure's documentation pointer. Anything else keeps dev's decision. Whitespace is collapsed and prose capped at 2,000 characters, so matching is linear. --- src/combos/failover.ts | 203 ++++++++++-------- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 43 +++- ...uter-combo-failover-classification.test.ts | 7 +- 4 files changed, 163 insertions(+), 92 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 4a15d67fc6c..9bff329a1df 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1004,60 +1004,65 @@ const MODERATION_COMPATIBLE_VALUES = new Set([ "invalid_request_error", "error", "ai_apicallerror", "responsibleaipolicyviolation", "400", "422", ]); -/** Keys whose values are structured verdicts rather than prose. */ -// Keys are compared folded: lowercase with underscores and hyphens removed (errorCode, error_code). -const MODERATION_VERDICT_KEYS = new Set(["code", "type", "errorcode", "errortype", "reason", "kind", "category"]); +/** Keys whose values are structured verdicts; folded (lowercase, underscores and hyphens removed). */ +const MODERATION_VERDICT_KEYS = new Set(["code", "type", "errorcode", "errortype", "reason"]); + +/** Fold a key for comparison: lowercase, underscores and hyphens removed. */ +function foldKey(key: string): string { + return key.toLowerCase().replace(/[_-]/g, ""); +} /** Keys that state an HTTP status; any stated status other than 400/422 is evidence of its own. */ const MODERATION_STATUS_KEYS = new Set(["status", "statuscode", "httpstatus", "httpstatuscode"]); -/** Keys whose string values are the body's prose. */ -const MODERATION_PROSE_KEYS = new Set(["message", "error", "detail", "details", "msg", "description", "errormessage"]); +/** Identifier fields a refusal envelope may carry: a bare token or number, never prose. */ +const MODERATION_ID_KEYS = new Set(["id", "requestid", "object", "model", "name", "sequencenumber", "createdat"]); /** - * Every prose string beside a coded verdict must itself describe the moderation refusal. A body - * that also says something else (a parameter error, a missing model, an inactive account) carries - * evidence of its own, so the verdict is not the whole story and the body keeps dev's decision. + * Keys a gateway's routing metadata (providerMetadata) must not use: a field that names a state, + * status, code or safety verdict is evidence of its own, so only plain routing tokens pass. */ -const MODERATION_PROSE = /moderat|content[\s_-]*(?:filter|management|safety)|\bfiltered\b|responsible[\s_-]*ai/i; +const ROUTING_METADATA_EVIDENCE_KEY = /safety|violation|block|polic|account|suspend|disabl|deactivat|ban|status|state|error|code|reason|quota|billing|limit|credit|payment/i; -/** - * Clauses a refusal may carry beside its verdict without adding evidence of its own: the category - * list, the advice to change the prompt and retry, and the pointer to documentation. - */ -const MODERATION_PROSE_FOLLOW_UP = - /^(?:categor(?:y|ies)\b.*|(?:please\s+)?(?:modify|revise|rephrase|change|adjust|edit|update)\s+(?:your|the)\s+(?:prompt|input|request|message|content)s?\b.*|(?:please\s+)?(?:try|retry)(?:\s+again)?\b.*|.*\b(?:learn\s+more|more\s+information|documentation)\b.*)$/i; +/** Azure's per-category filter results: the moderation verdict's own detail, read only for codes. */ +const MODERATION_RESULT_KEY = /^(?:contentfilterresults?|promptfilterresults)$/; /** - * Whether a prose string only describes the moderation refusal. Each clause (split at sentence - * ends, semicolons, commas and joining words) must name the refusal or be a known follow-up, so a - * string that also reports an account, model, parameter or billing problem keeps dev's decision. + * Response-object fields of a Responses terminal event (`response.failed`) that echo the caller's + * request or report usage. They say nothing about the failure and are not read. */ -function describesOnlyModeration(text: string): boolean { - const clauses = text.split(/(?<=[.!?;,])\s+|\s+(?:and|but|however|while|although|because)\s+|\n+/i) - .map(clause => clause.trim()).filter(clause => clause !== ""); - return clauses.some(clause => MODERATION_PROSE.test(clause)) - && clauses.every(clause => MODERATION_PROSE.test(clause) || MODERATION_PROSE_FOLLOW_UP.test(clause)); -} - -/** Fold a key for comparison: lowercase, underscores and hyphens removed. */ -function foldKey(key: string): string { - return key.toLowerCase().replace(/[_-]/g, ""); -} - -/** Response-object fields that echo the caller's request rather than describe the failure. */ const RESPONSE_REQUEST_ECHO_KEYS = new Set([ - "instructions", "tools", "tool_choice", "text", "reasoning", "input", "output", "prompt", "include", - "user", "safety_identifier", "prompt_cache_key", "previous_response_id", "conversation", + "instructions", "tools", "toolchoice", "text", "reasoning", "input", "output", "prompt", "include", + "user", "safetyidentifier", "promptcachekey", "promptcacheretention", "previousresponseid", "conversation", + "temperature", "topp", "toplogprobs", "maxoutputtokens", "maxtoolcalls", "paralleltoolcalls", "truncation", + "store", "servicetier", "background", "usage", ]); /** - * Structured safety or policy flags (`safety_violation`, `blocked`, `policy_violation`). A - * flag is the upstream's own safety verdict; only Azure's per-category content-filter results, - * which describe the moderation verdict itself, may carry such fields. + * The whole prose a moderation refusal may carry, after whitespace is collapsed: one verdict + * sentence (the gateway's own refusal copy, OpenAI's and Azure's included), optionally followed by + * the category list, the advice to change the prompt and retry, and Azure's documentation pointer. + * Any other wording (an account, model, parameter or billing problem, in any phrasing) does not + * match, so the body keeps dev's decision; only known shapes are admitted. */ -const SAFETY_FLAG_KEY = /safety|violation|block|polic|account|suspend|disabl|deactivat|banned/i; -const MODERATION_RESULT_KEY = /^content_filter_results?$|^prompt_filter_results?$/i; +const MODERATION_REFUSAL_PROSE = new RegExp("^(?:" + + "(?:content )?(?:refused|flagged|blocked|rejected|filtered)(?: by (?:the )?(?:moderation|content filter))?\\.?" + + "|(?:(?:the|your) )?(?:input|request|prompt|content|message|response|output)(?: (?:was|has been|is))?" + + " (?:refused|flagged|blocked|rejected|filtered)(?: by (?:the )?(?:moderation|content filter)(?: system)?)?\\.?" + + "|the response was filtered due to the prompt triggering azure openai'?s content management policy\\.?" + + "|(?:your )?(?:input|request|prompt|content) violated (?:the )?content (?:safety|management|filtering) polic(?:y|ies)\\.?" + + ")(?:" + + " categor(?:y|ies): [a-z][a-z/_-]*(?:(?:,? and |, )[a-z][a-z/_-]*)*\\.?" + + "| please (?:modify|revise|rephrase|change|adjust|edit) (?:your|the) (?:prompt|input|request|message)(?: and (?:retry|try again))?\\.?" + + "| (?:please )?(?:retry|try again)(?: later)?\\.?" + + "| to learn more about our content filtering policies please read our documentation: https://[^ ]+" + + ")*$", "i"); + +/** Whether a prose string is a known moderation refusal copy and nothing else (bounded, linear). */ +function isModerationRefusalProse(text: string): boolean { + if (text.length > 2_000) return false; + return MODERATION_REFUSAL_PROSE.test(text.replace(/\s+/g, " ").trim()); +} /** * Safety-system wording in any of its upstream phrasings ("rejected as a result of our safety @@ -1133,67 +1138,91 @@ function moderationEnvelope(message: string): unknown { } /** - * Walk every structured verdict field (bounded). Returns whether a moderation code was seen, or - * null when any field states something else, a field holds a non-string value we cannot vouch - * for, or the body is deeper or wider than a refusal envelope. + * Walk a refusal envelope (bounded) against the closed shape of a moderation refusal. Returns whether + * a moderation code was seen, or null when the body carries anything else: an unknown field, prose + * that is not refusal copy, a param outside the prompt or input, a stated status other than 400/422, + * an incompatible code, or more nesting than a refusal envelope has. Only known shapes hop; anything + * unrecognized keeps dev's decision. */ function moderationVerdictFields(payload: unknown): boolean | null { let seen = false; let nodes = 0; - const visit = (value: unknown, depth: number, insideFilterResult: boolean): boolean => { + const verdict = (field: unknown, key: string, depth: number): boolean => { + if (field === null || field === undefined) return true; + // A Responses terminal event names itself in its root `type` (`response.failed`): that is the + // transport's label for the carrier, not the upstream's verdict. + if (depth === 0 && key === "type" && typeof field === "string" && /^response\.[a-z_.]+$/.test(field)) return true; + if (typeof field !== "string" && typeof field !== "number") return false; + const normalized = normalizedFailureCode(field); + if (normalized === "") return typeof field === "string"; + if (!MODERATION_COMPATIBLE_VALUES.has(normalized)) return false; + if (MODERATION_REFUSAL_CODES.has(normalized)) seen = true; + return true; + }; + // Inside Azure's filter results only codes are read; the per-category flags are the verdict itself. + const visitFilterResult = (value: unknown, depth: number): boolean => { if (++nodes > 512 || depth > 8) return false; - if (Array.isArray(value)) { - // A string item is prose like any other field (`details: ["..."]`). - return value.every(item => typeof item === "string" - ? insideFilterResult || !/\s/.test(item.trim()) || describesOnlyModeration(item) - : visit(item, depth + 1, insideFilterResult)); - } + if (Array.isArray(value)) return value.every(item => visitFilterResult(item, depth + 1)); if (!value || typeof value !== "object") return true; + return Object.entries(value as Record).every(([key, field]) => + MODERATION_VERDICT_KEYS.has(foldKey(key)) ? verdict(field, key, depth) : visitFilterResult(field, depth + 1)); + }; + // A gateway's routing metadata: plain tokens, numbers and flags only, under keys that name no state. + const visitRoutingMetadata = (value: unknown, depth: number): boolean => { + if (++nodes > 512 || depth > 8) return false; + if (value === null || typeof value === "number" || typeof value === "boolean") return true; + if (typeof value === "string") return !/\s/.test(value); + if (Array.isArray(value)) return value.every(item => visitRoutingMetadata(item, depth + 1)); + if (typeof value !== "object") return false; + return Object.entries(value as Record).every(([key, field]) => + !ROUTING_METADATA_EVIDENCE_KEY.test(key) && visitRoutingMetadata(field, depth + 1)); + }; + const visit = (value: unknown, depth: number, responseObject: boolean): boolean => { + if (++nodes > 512 || depth > 8) return false; + if (!value || typeof value !== "object" || Array.isArray(value)) return false; const record = value as Record; - // A Responses terminal event (`response.failed`) carries the whole response object, which echoes - // the caller's request (instructions, tools, text format...). Those echoes say nothing about the - // verdict; every other field of the response object is still read. const responseCarrier = depth === 0 && typeof record.type === "string" && /^response\.[a-z_.]+$/.test(record.type); for (const [key, field] of Object.entries(record)) { - if (responseCarrier && key === "response" && field && typeof field === "object" && !Array.isArray(field)) { - const verdictFields = Object.fromEntries(Object.entries(field as Record) - .filter(([responseKey]) => !RESPONSE_REQUEST_ECHO_KEYS.has(responseKey))); - if (!visit(verdictFields, depth + 1, insideFilterResult)) return false; - continue; - } - // A param names what the filter inspected: the prompt or input. Any other param, on any - // record, marks a validation error. - if (key.toLowerCase() === "param" && typeof field === "string" && field.trim() !== "" && !isAllowedModerationParam(field)) { - return false; - } - if (MODERATION_STATUS_KEYS.has(foldKey(key))) { + const folded = foldKey(key); + if (responseObject && RESPONSE_REQUEST_ECHO_KEYS.has(folded)) continue; + if (MODERATION_VERDICT_KEYS.has(folded)) { + if (!verdict(field, key, depth)) return false; + } else if (MODERATION_STATUS_KEYS.has(folded)) { + if (field === null) continue; const stated = typeof field === "number" ? field : typeof field === "string" && /^\d{3}$/.test(field.trim()) ? Number(field) : undefined; - if (stated !== undefined && stated >= 400 && stated !== 400 && stated !== 422) return false; - if (typeof field !== "object" || field === null) continue; - } - if (!insideFilterResult && SAFETY_FLAG_KEY.test(key)) return false; - // Prose beside the verdict, under any key (message, hint, note, metadata.raw...), must itself - // describe the moderation refusal; a bare token (an id, an enum) is not prose. - if (typeof field === "string" && field.trim() !== "" && !insideFilterResult - && key.toLowerCase() !== "param" && !MODERATION_VERDICT_KEYS.has(foldKey(key)) - && (MODERATION_PROSE_KEYS.has(foldKey(key)) || /\s/.test(field.trim())) - && !describesOnlyModeration(field)) { + if (stated !== undefined ? stated !== 400 && stated !== 422 + : !(typeof field === "string" && /^(?:failed|incomplete|error)$/i.test(field.trim()))) return false; + } else if (folded === "message") { + if (field !== null && field !== "" && !(typeof field === "string" && isModerationRefusalProse(field))) return false; + } else if (folded === "param") { + // A param names what the filter inspected: the prompt or input. Any other param is a + // validation error. + if (field !== null && typeof field === "object" && !Array.isArray(field)) { + // An SDK error object in the param slot (an AI gateway's wrapped error) is read as a record. + if (!visit(field, depth + 1, false)) return false; + } else if (field !== null && field !== "" && !(typeof field === "string" && isAllowedModerationParam(field))) return false; + } else if (MODERATION_ID_KEYS.has(folded)) { + if (field !== null && typeof field !== "number" && !(typeof field === "string" && !/\s/.test(field))) return false; + } else if (folded === "isretryable") { + if (field !== null && typeof field !== "boolean") return false; + } else if (depth === 0 && folded === "providermetadata") { + if (!visitRoutingMetadata(field, depth + 1)) return false; + } else if (MODERATION_RESULT_KEY.test(folded)) { + if (!visitFilterResult(field, depth + 1)) return false; + } else if (folded === "error") { + if (field === null) continue; + if (typeof field === "string" ? !isModerationRefusalProse(field) : !visit(field, depth + 1, false)) return false; + } else if (folded === "innererror" || (responseObject && folded === "incompletedetails")) { + if (field !== null && !visit(field, depth + 1, false)) return false; + } else if (responseObject && folded === "metadata") { + // The caller's own request metadata; a refusal never needs it, so only an empty one passes. + if (field !== null && !(typeof field === "object" && !Array.isArray(field) && Object.keys(field).length === 0)) return false; + } else if (responseCarrier && key === "response") { + if (!visit(field, depth + 1, true)) return false; + } else { return false; } - if (MODERATION_VERDICT_KEYS.has(foldKey(key))) { - if (field === null || field === undefined) continue; - // A Responses terminal event names itself in its root `type` (`response.failed`): that is - // the transport's label for the carrier, not the upstream's verdict. - if (depth === 0 && key === "type" && typeof field === "string" && /^response\.[a-z_.]+$/.test(field)) continue; - const normalized = normalizedFailureCode(typeof field === "number" || typeof field === "string" ? field : null); - if (normalized === "" && typeof field !== "string") return false; - if (normalized === "") continue; - if (!MODERATION_COMPATIBLE_VALUES.has(normalized)) return false; - if (MODERATION_REFUSAL_CODES.has(normalized)) seen = true; - continue; - } - if (!visit(field, depth + 1, insideFilterResult || MODERATION_RESULT_KEY.test(key))) return false; } return true; }; @@ -1248,7 +1277,7 @@ export function isProviderModerationRefusal( if (envelope === undefined) { // Plain text carries no structured fields; only the caller's code can state the verdict, and a // body that looks like JSON but does not parse (cut or malformed) never hops. - return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message) && describesOnlyModeration(message); + return callerStatesModeration && !/^\s*(?:data:\s*)?[[{]/.test(message) && isModerationRefusalProse(message); } // An envelope root that states a verdict code of its own (invalid_request_error, origin, // context length, cancellation...) keeps it, as everywhere else in this file. diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index d33c17a4f6c..c0fef038511 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only: a `moderation_refusal` or `content_filter` code (the caller's or one in the body), every other `code` and `type` anywhere in the body on a short compatible list, and no `param` outside the prompt/input paths on any record, the coded one included; verdict-like keys (`code`, `type`, `error_code`, `error_type`, `reason`, `kind`, `category`) must all be compatible, a stated `status`/`status_code`/`http_status` other than 400/422 and any safety or policy flag outside Azure's per-category filter results veto, and a Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`...; `metadata` is still read). Every prose string in the body, under any key or inside an array (`message`, `detail`, `hint`, `metadata.raw`, `details[]`...), must only describe a moderation or content-filter refusal: each clause names the refusal or is a known follow-up (categories, change the prompt and retry, documentation); a parameter error, missing model or inactive account beside the code keeps dev's decision. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only and only for the known shape of a refusal: a `moderation_refusal` or `content_filter` code (the caller's or one in the body) and nothing else. The walk (`moderationVerdictFields`) admits a closed set of fields: verdict codes and types on a short compatible list, a stated status of 400/422 (or a Responses `failed`/`incomplete` state), a `param` naming the prompt or input (or an AI SDK error object read as a record), bare identifiers (`id`, `request_id`, `model`, `name`...), `innererror`, Azure's per-category filter results (read only for codes), and a gateway's `providerMetadata` holding plain routing tokens under keys that name no state. Any other field stops. Prose (`message`, a string `error`, plain text with a caller's code) must be known refusal copy as a whole, after whitespace is collapsed: one verdict sentence (OpenAI's "The input was refused by moderation", Azure's content-management sentence, a bare "filtered"...) optionally followed by the category list, the advice to change the prompt and retry, and Azure's documentation pointer; anything else, an account, model, parameter or billing problem in any phrasing, keeps dev's decision. A Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`, `usage`...); its `metadata` must be empty. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index d1751161007..b37c1980eab 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -390,13 +390,45 @@ describe("every prose field beside a coded verdict must describe the moderation JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry.", innererror: { code: "ResponsibleAIPolicyViolation" } } }), JSON.stringify({ error: { code: "moderation_refusal", message: "Your request was flagged by moderation." } }), coded("The input was refused by moderation.", { request_id: "req_abc123" }), - coded("The input was refused by moderation.", { param: "input", details: ["harassment"] }), + coded("The input was refused by moderation.", { param: "input" }), + coded("The input was refused by moderation. Categories: sexual, hate. Please modify your prompt and retry."), + coded("The input was refused by moderation. Categories: hate, harassment, self-harm."), + coded("The input was refused by moderation. Categories: sexual and hate."), + coded("filtered"), + coded("Content filtered by moderation."), JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry. To learn more about our content filtering policies please read our documentation: https://go.microsoft.com/fwlink/?linkid=2198766" } }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: true }); } }); + test("only known refusal copy and known fields pass beside a coded verdict", async () => { + for (const body of [ + // Evidence joined by separators other than a sentence break. + coded("Refused by moderation \u2014 your account is past due."), + coded("Refused by moderation: your account is past due."), + coded("Filtered by moderation (your account is past due)."), + coded("Filtered by moderation / your account is past due."), + coded("Filtered by moderation\tyour account is past due."), + coded("Filtered by moderation;your account is past due."), + coded("Filtered by moderation so your account is past due."), + // Follow-up clauses that carry evidence of their own. + coded("Content filtered by moderation. Learn more: the model gpt-9 does not exist."), + coded("Content filtered by moderation. For more information about your suspended account."), + coded("Content filtered by moderation. Retry with model gpt-9 which does not exist."), + coded("Content filtered by moderation. Please modify your input for the suspended account."), + coded("The input was refused by moderation. Categories: model gpt-9 does not exist."), + // Bare tokens under fields a refusal does not carry. + coded("The input was refused by moderation.", { metadata: { account: "suspended" } }), + coded("The input was refused by moderation.", { state: "past_due" }), + coded("The input was refused by moderation.", { account_status: "disabled" }), + ]) { + expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: false }); + const failure = await consumeComboFailure(new Response(body, { status: 400 })); + expect(comboFailureCooldownScope(400, failure.classificationText, { code: failure.upstreamCode, providerModerationRefusal: failure.providerModerationRefusal })).not.toBe("none"); + } + }); + test("a response.failed carrier's echoed instructions and tools do not veto its coded refusal", async () => { const frame = { type: "response.failed", @@ -428,6 +460,7 @@ describe("every prose field beside a coded verdict must describe the moderation { metadata: { raw: "Your account is past due" } }, { metadata: { httpStatus: 503 } }, { metadata: { safety_violation: true } }, + { metadata: { client: "my app v2" } }, ]) { const frame = { type: "response.failed", response: { status: "failed", ...extra, error } }; const result = await preflightComboStreamResponse(new Response( @@ -439,4 +472,12 @@ describe("every prose field beside a coded verdict must describe the moderation expect({ extra, scope }).not.toEqual({ extra, scope: "none" }); } }); + + test("a long whitespace run in upstream prose is judged in linear time", () => { + // The clause splitter once backtracked across whitespace runs: about 2 s for this body. + const body = JSON.stringify({ error: { code: "content_filter", message: "moderation" + " ".repeat(60_000) + "x" } }); + const started = performance.now(); + expect(isProviderModerationRefusal(400, body, undefined, true)).toBe(false); + expect(performance.now() - started).toBeLessThan(250); + }); }); diff --git a/tests/routing/router-combo-failover-classification.test.ts b/tests/routing/router-combo-failover-classification.test.ts index 912655ac796..1b7351c4894 100644 --- a/tests/routing/router-combo-failover-classification.test.ts +++ b/tests/routing/router-combo-failover-classification.test.ts @@ -1421,7 +1421,8 @@ describe("provider moderation refusal", () => { const body = JSON.stringify({ error: { message: directValidation } }); expect(isProviderModerationRefusal(400, body)).toBe(false); - expect(isProviderModerationRefusal(400, directValidation, "content_filter")).toBe(true); + // A validation error is evidence of its own: even with a caller's code it is not refusal copy. + expect(isProviderModerationRefusal(400, directValidation, "content_filter")).toBe(false); }); test("accepts indexed content parameter paths in coded moderation refusals", () => { @@ -1477,12 +1478,12 @@ describe("provider moderation refusal", () => { }); test("a coded refusal over 500 characters hops on the whole body, never on the cut text", async () => { - const padding = "x".repeat(600); + const requestId = "req_" + "x".repeat(600); const raw = JSON.stringify({ error: { code: "content_filter", message: "The input was refused by moderation. Categories: harassment.", - padding, + request_id: requestId, }, }); // A cut envelope does not parse, so it carries no structured verdict of its own. From 4c3e753466e31038e717ab25f30f1c132af04bde Mon Sep 17 00:00:00 2001 From: lidge-jun <243035832+lidge-jun@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:09:28 +0900 Subject: [PATCH 29/29] fix(combo): read Azure filter results in their own shape only Inside content_filter_result(s) and prompt_filter_results the walk read codes but accepted any other field, so a note, an account token, a status or an error object placed there let a coded refusal hop without cooling. Filter results now pass only as category objects of flags and numbers, a severity of safe/low/medium/high, bare id/url/license tokens and compatible codes; any other text, or a key that names a state, status, error or message, stops. --- src/combos/failover.ts | 24 +++++++++++++++---- structure/providers-and-adapters.md | 2 +- .../combo-failure-transport-parity.test.ts | 18 ++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 9bff329a1df..dcbf75d4b77 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1024,8 +1024,14 @@ const MODERATION_ID_KEYS = new Set(["id", "requestid", "object", "model", "name" */ const ROUTING_METADATA_EVIDENCE_KEY = /safety|violation|block|polic|account|suspend|disabl|deactivat|ban|status|state|error|code|reason|quota|billing|limit|credit|payment/i; -/** Azure's per-category filter results: the moderation verdict's own detail, read only for codes. */ +/** + * Azure's per-category filter results: the moderation verdict's own detail. Only its own shape + * passes: category objects of flags and numbers, a `severity` of safe/low/medium/high, bare + * `id`/`url`/`license` tokens, and compatible codes. Any other text, or a key that names a + * state, status, error or message, is evidence of its own. + */ const MODERATION_RESULT_KEY = /^(?:contentfilterresults?|promptfilterresults)$/; +const FILTER_RESULT_EVIDENCE_KEY = /account|status|state|quota|billing|payment|credit|limit|error|message|reason|note|description|detail(?!s$)/i; /** * Response-object fields of a Responses terminal event (`response.failed`) that echo the caller's @@ -1159,13 +1165,21 @@ function moderationVerdictFields(payload: unknown): boolean | null { if (MODERATION_REFUSAL_CODES.has(normalized)) seen = true; return true; }; - // Inside Azure's filter results only codes are read; the per-category flags are the verdict itself. const visitFilterResult = (value: unknown, depth: number): boolean => { if (++nodes > 512 || depth > 8) return false; + if (value === null || typeof value === "boolean" || typeof value === "number") return true; if (Array.isArray(value)) return value.every(item => visitFilterResult(item, depth + 1)); - if (!value || typeof value !== "object") return true; - return Object.entries(value as Record).every(([key, field]) => - MODERATION_VERDICT_KEYS.has(foldKey(key)) ? verdict(field, key, depth) : visitFilterResult(field, depth + 1)); + if (typeof value !== "object") return false; + return Object.entries(value as Record).every(([key, field]) => { + const folded = foldKey(key); + if (MODERATION_VERDICT_KEYS.has(folded)) return verdict(field, key, depth); + if (!/^[a-z][a-z0-9_]*$/i.test(key) || FILTER_RESULT_EVIDENCE_KEY.test(key)) return false; + if (typeof field === "string") { + return folded === "severity" ? /^(?:safe|low|medium|high)$/i.test(field) + : (folded === "id" || folded === "url" || folded === "license") && !/\s/.test(field); + } + return visitFilterResult(field, depth + 1); + }); }; // A gateway's routing metadata: plain tokens, numbers and flags only, under keys that name no state. const visitRoutingMetadata = (value: unknown, depth: number): boolean => { diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index c0fef038511..cb8c23e618e 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -587,7 +587,7 @@ Account-row projection is loaded with `import()` after each account probe in `sr ## Combo model refusal evidence -`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only and only for the known shape of a refusal: a `moderation_refusal` or `content_filter` code (the caller's or one in the body) and nothing else. The walk (`moderationVerdictFields`) admits a closed set of fields: verdict codes and types on a short compatible list, a stated status of 400/422 (or a Responses `failed`/`incomplete` state), a `param` naming the prompt or input (or an AI SDK error object read as a record), bare identifiers (`id`, `request_id`, `model`, `name`...), `innererror`, Azure's per-category filter results (read only for codes), and a gateway's `providerMetadata` holding plain routing tokens under keys that name no state. Any other field stops. Prose (`message`, a string `error`, plain text with a caller's code) must be known refusal copy as a whole, after whitespace is collapsed: one verdict sentence (OpenAI's "The input was refused by moderation", Azure's content-management sentence, a bare "filtered"...) optionally followed by the category list, the advice to change the prompt and retry, and Azure's documentation pointer; anything else, an account, model, parameter or billing problem in any phrasing, keeps dev's decision. A Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`, `usage`...); its `metadata` must be empty. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. +`src/combos/failover.ts` recognizes a bounded, exact HTTP 400 plan/model refusal, not credential or entitlement evidence. It also permits OpenRouter's exact opaque numeric-400 wrapper (`Provider returned error` with `metadata.raw: "ERROR"`, without `type`) to try another combo target. Typed uncoded 400s, malformed `response_format` schema errors and other terminal `invalid_request_error` cases stop and cool the target. An envelope root that states a terminal code of its own stops too (`origin_rejected`, `content_policy_violation`, `context_length_exceeded`, `invalid_request_error`, the shared terminal-refusal set and the non-replayable post-send codes, not a blanket inequality); a root that states only an ordinary hop signal adds no verdict, and an empty code counts as absent. The root code and the record it wraps are both consulted, so a hop needs each to authorize one and a root hard stop is never hidden behind a retryable nested record. Request-local target incompatibilities recognized by `isRequestLocalTargetIncompatibility` and `isResponseFormatCapabilityRefusal` may advance without cooling. So may a provider or gateway moderation verdict on HTTP 400/422 (`isProviderModerationRefusal`), granted on structured evidence only and only for the known shape of a refusal: a `moderation_refusal` or `content_filter` code (the caller's or one in the body) and nothing else. The walk (`moderationVerdictFields`) admits a closed set of fields: verdict codes and types on a short compatible list, a stated status of 400/422 (or a Responses `failed`/`incomplete` state), a `param` naming the prompt or input (or an AI SDK error object read as a record), bare identifiers (`id`, `request_id`, `model`, `name`...), `innererror`, Azure's per-category filter results in their own shape only (category objects of flags and numbers, a `severity` of safe/low/medium/high, bare `id`/`url`/`license` tokens and compatible codes; any other text, or a key naming a state, status, error or message, stops), and a gateway's `providerMetadata` holding plain routing tokens under keys that name no state. Any other field stops. Prose (`message`, a string `error`, plain text with a caller's code) must be known refusal copy as a whole, after whitespace is collapsed: one verdict sentence (OpenAI's "The input was refused by moderation", Azure's content-management sentence, a bare "filtered"...) optionally followed by the category list, the advice to change the prompt and retry, and Azure's documentation pointer; anything else, an account, model, parameter or billing problem in any phrasing, keeps dev's decision. A Responses event's own `response.*` type is read as the carrier, whose response object is read without the fields that echo the caller's request (`instructions`, `tools`, `text`, `reasoning`, `input`, `output`, `usage`...); its `metadata` must be empty. Prose alone never hops (owner decision on #8428). It never outranks a stop: an envelope root verdict code, a hard-refusal token in any spelling (JSON-escaped, camelCase, or with hyphens, dots, slashes or spaces for underscores), safety or terminal-refusal copy, a malformed `response_format` diagnostic, subscription-gate wording, or any text the shared `classifyError` files more specifically than a generic invalid request keeps the terminal decision and its cooldown, and quota, billing or account-state evidence (including such wording beside a moderation code) still cools the provider. The verdict needs the whole body: `consumeComboFailure`, the responses stream preflight, the Messages native preflight and `policy-fallback.ts` judge the complete bounded body or event and pass `providerModerationRefusal`; a preflight that rebuilds a slim envelope attaches the whole event as combo failure evidence whenever either reads as a moderation verdict, and an event that states a non-400/422 status of its own is never a moderation refusal; and without it a classification text of 500 characters or more (possibly cut) never hops. `src/server/responses/core-combo-failure.ts` captures its finite classification before display truncation; `src/server/responses/combo-stream-preflight.ts` preserves it before rebuilding terminal SSE errors. `src/server/responses/core-options.ts` carries the value to the combo decision. An inspected non-match diff --git a/tests/routing/combo-failure-transport-parity.test.ts b/tests/routing/combo-failure-transport-parity.test.ts index b37c1980eab..ba41bc92443 100644 --- a/tests/routing/combo-failure-transport-parity.test.ts +++ b/tests/routing/combo-failure-transport-parity.test.ts @@ -354,6 +354,16 @@ describe("provider evidence and status-less response.failed frames beside a code describe("every prose field beside a coded verdict must describe the moderation refusal", () => { const coded = (message: string, extra: Record = {}) => JSON.stringify({ error: { type: "invalid_request_error", code: "content_filter", message, ...extra } }); + const azureResults = { + hate: { filtered: true, severity: "high" }, self_harm: { filtered: false, severity: "safe" }, + jailbreak: { filtered: false, detected: false }, + custom_blocklists: { filtered: false, details: [{ filtered: false, id: "list1" }] }, + }; + const azureWith = (extra: Record) => JSON.stringify({ error: { + code: "content_filter", + message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry.", + innererror: { code: "ResponsibleAIPolicyViolation", content_filter_result: { ...azureResults, ...extra } }, + } }); test("parameter, model and account prose beside a moderation code keeps dev's decision", async () => { for (const body of [ @@ -396,6 +406,7 @@ describe("every prose field beside a coded verdict must describe the moderation coded("The input was refused by moderation. Categories: sexual and hate."), coded("filtered"), coded("Content filtered by moderation."), + azureWith({}), JSON.stringify({ error: { code: "content_filter", message: "The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry. To learn more about our content filtering policies please read our documentation: https://go.microsoft.com/fwlink/?linkid=2198766" } }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: true }); @@ -422,6 +433,13 @@ describe("every prose field beside a coded verdict must describe the moderation coded("The input was refused by moderation.", { metadata: { account: "suspended" } }), coded("The input was refused by moderation.", { state: "past_due" }), coded("The input was refused by moderation.", { account_status: "disabled" }), + // Azure's filter results carry only their own shape. + azureWith({ note: "Your account is past due" }), + azureWith({ hate: { filtered: true, severity: "high", note: "account past due" } }), + azureWith({ account: "suspended" }), + azureWith({ hate: { filtered: true, severity: "past_due" } }), + azureWith({ error: { code: "content_filter_error", message: "The contents are not filtered" } }), + azureWith({ hate: { filtered: true, severity: "high", status: 429 } }), ]) { expect({ body, verdict: isProviderModerationRefusal(400, body, undefined, true) }).toEqual({ body, verdict: false }); const failure = await consumeComboFailure(new Response(body, { status: 400 }));