diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 435e83f1a90..50bd65699d7 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -499,7 +499,8 @@ MiMo model Command Code serves. limits inform context windows, conservatively combined with shipped limits when evidence is partial. Accounts that have not served have no list evidence yet. - Participates in multi-account rotation. With two stored accounts, a request-rate refusal - cools the refused account briefly; an exact monthly-quota refusal on HTTP 400 or 429 + cools the refused account briefly; an exact `MONTHLY_REQUEST_COUNT` or + `OVERAGE_REQUEST_LIMIT_EXCEEDED` refusal on HTTP 400 or 429 excludes it until the observed reset or evidence expiry. A confirmed HTTP 403 suspension quarantines that account in process, while an ordinary 403 does not rotate. Reactive rotation remains available when `oauthAccountFailover.enabled` is false. Refusal-aware diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index 46a343e2b85..8c0c237f941 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -787,6 +787,8 @@ separately billed sends add to the request spend. Credits are never estimated fr With two or more Kiro accounts logged in, request-rate, confirmed monthly-quota, and confirmed suspension refusals can rotate before output. Monthly exhaustion excludes only that login until reset or evidence expiry; completed service clears an older verdict. +Quota refusals include the exact `MONTHLY_REQUEST_COUNT` and `OVERAGE_REQUEST_LIMIT_EXCEEDED` +reasons on HTTP 400 or 429. Plans allowing overage can keep serving after the monthly gauge reaches 100%. Reactive rotation remains available when proactive account preference is off. Accounts are added one at a time — `ocx account login kiro` hands off to the Kiro CLI and appends the new account to the pool. In the dashboard, Login and Add account also offer Builder ID, Google, and GitHub device login alongside the Kiro CLI choice; native device login adds an account without signing the CLI out. Users of one organization (IAM Identity Center) profile share its profile ARN, so a Kiro CLI account is identified by that profile together with the email `kiro-cli whoami` reports for the session. Two users of the same organization therefore stay separate accounts, and signing the same user in again updates that user's account. When `kiro-cli whoami` reports no user, the profile alone identifies the account as before. If that profile already has an account with a known user, such a login only refreshes the account whose stored Kiro CLI session it continues; any other login is refused instead of replacing an account, and an Add account login switches the Kiro CLI back to its previous session. Run `kiro-cli whoami` to confirm the signed-in user, then retry. Reauthenticating an account saved before its user was known records the user that signs in, unless that user already has another account of the same profile. diff --git a/src/adapters/kiro-refusal.ts b/src/adapters/kiro-refusal.ts index fbfc0ee784c..58c8fdd0016 100644 --- a/src/adapters/kiro-refusal.ts +++ b/src/adapters/kiro-refusal.ts @@ -13,6 +13,10 @@ const SUSPENSION_WORDS = [ "locked your account", "locked it as a", ] as const; +/** + * Classify a Kiro HTTP error for account rotation and quota or suspension evidence. + * Reads bounded, top-level JSON fields; unrecognized 429 bodies retain rate recovery. + */ export function classifyKiroRefusal(status: number, bodyText: string): KiroRefusal { if (Buffer.byteLength(bodyText, "utf8") > BOUNDED_BODY_MAX_BYTES) return { kind: status === 429 ? "rate" : "other" }; @@ -25,7 +29,8 @@ export function classifyKiroRefusal(status: number, bodyText: string): KiroRefus message = (row as Record).message; } } catch { /* Unrecognised bodies cannot establish an account verdict. */ } - if ((status === 400 || status === 429) && reason === "MONTHLY_REQUEST_COUNT") + if ((status === 400 || status === 429) + && (reason === "MONTHLY_REQUEST_COUNT" || reason === "OVERAGE_REQUEST_LIMIT_EXCEEDED")) return { kind: "monthly_quota" }; // A 400 whose own validator calls the body improperly formed is not proof that the request is // impossible: Kiro answers this shape when the identity a request was routed under cannot serve diff --git a/structure/providers/kiro.md b/structure/providers/kiro.md index e0e26d4ff12..0dc3a69f078 100644 --- a/structure/providers/kiro.md +++ b/structure/providers/kiro.md @@ -74,7 +74,8 @@ rotation reaches them. The public catalog advertises only observed IDs made of p characters (no `/` the router would have to decode), at most 64 across the roster; every observed ID still informs routing preference. -`src/adapters/kiro-refusal.ts` recognizes an exact monthly reason on HTTP 400/429 and a +`src/adapters/kiro-refusal.ts` recognizes exact `MONTHLY_REQUEST_COUNT` and +`OVERAGE_REQUEST_LIMIT_EXCEEDED` reasons on HTTP 400/429 as account quota exhaustion, and a confirmed suspension on HTTP 403; ordinary 400/403 remains an error without an account verdict. A 400 whose parsed body carries `ValidationException` (as `reason`, or inside `message`) and the message "Improperly formed request" is the `profile` kind: the account layer rotates to a diff --git a/tests/providers/kiro/kiro-refusal.test.ts b/tests/providers/kiro/kiro-refusal.test.ts index 6b0e6f2353b..b1740dd1a46 100644 --- a/tests/providers/kiro/kiro-refusal.test.ts +++ b/tests/providers/kiro/kiro-refusal.test.ts @@ -14,6 +14,20 @@ describe("Kiro refusal classification", () => { expect(classified(403, { reason: "MONTHLY_REQUEST_COUNT" }).kind).toBe("other"); }); + test("kiro refusal: exhausted overage allowance is account quota, not a bad request", () => { + const body = { __type: "com.amazon.kiro.runtimeservice#ServiceQuotaExceededException", + reason: "OVERAGE_REQUEST_LIMIT_EXCEEDED", message: "You have reached the limit for overages." }; + for (const status of [400, 429]) + expect(classified(status, body)).toEqual({ kind: "monthly_quota" }); + for (const status of [401, 403, 500]) + expect(classified(status, body).kind).toBe("other"); + for (const unrelated of [ + { reason: "REQUEST_BODY_INVALID", message: "OVERAGE_REQUEST_LIMIT_EXCEEDED" }, + { error: body }, { metadata: body.reason }, { reason: `${body.reason}_UNKNOWN` }, + { ...body, padding: "x".repeat(70_000) }, + ]) expect(classified(400, unrelated).kind).toBe("other"); + }); + test("kiro refusal: runtime and q-host suspension shapes require 403", () => { expect(classified(403, { reason: "TEMPORARILY_SUSPENDED" }).kind).toBe("suspended"); for (const message of ["temporarily suspended", "temporarily is suspended", diff --git a/tests/server/server-kiro-refusal-e2e.test.ts b/tests/server/server-kiro-refusal-e2e.test.ts index 7ea09925cdf..b3db26cfc3c 100644 --- a/tests/server/server-kiro-refusal-e2e.test.ts +++ b/tests/server/server-kiro-refusal-e2e.test.ts @@ -193,34 +193,42 @@ describe("Kiro refusal recovery through Responses", () => { } finally { ADAPTER_REGISTRY.kiro.create = originalCreate; await server?.stop(true); } }); - for (const [status, reason] of [[400, "MONTHLY_REQUEST_COUNT"], [403, "TEMPORARILY_SUSPENDED"], + for (const [status, reason] of [[400, "MONTHLY_REQUEST_COUNT"], + [400, "OVERAGE_REQUEST_LIMIT_EXCEEDED"], [429, "OVERAGE_REQUEST_LIMIT_EXCEEDED"], [403, "TEMPORARILY_SUSPENDED"], [429, "USER_REQUEST_RATE_EXCEEDED"]] as const) { - test(`streaming Kiro ${status} ${reason} rotates with paired bearer profile and region`, async () => { - const [a] = await seed(); - saveConfig(config()); - const observed: Array<{ url: string; auth: string | null; profile: string | null }> = []; - globalThis.fetch = (async (input, init) => { - const url = input instanceof Request ? input.url : String(input); - if (url.startsWith("https://runtime.") && url.endsWith(".kiro.dev/")) { - observed.push({ url, auth: new Headers(init?.headers).get("authorization"), - profile: new Headers(init?.headers).get("x-amzn-kiro-profile-arn") }); - if (observed.length === 1) return new Response(JSON.stringify({ reason }), { status }); - return answer("served by b"); - } - return realFetch(input, init); - }) as typeof fetch; - const server = startServer(0); - try { - const response = await post(server, true); - expect(response.status).toBe(200); - expect(await response.text()).toContain("served by b"); - expect(observed).toEqual([ - { url: "https://runtime.us-east-1.kiro.dev/", auth: "Bearer access-a", profile: aProfile }, - { url: "https://runtime.eu-west-1.kiro.dev/", auth: "Bearer access-b", profile: bProfile }, - ]); - if (status === 400) expect(kiroAccountEvidence(a!).exhausted).toBe(true); - } finally { await server.stop(true); } - }); + for (const stream of [false, true]) { + test(`${stream ? "streaming" : "buffered"} Kiro ${status} ${reason} rotates with paired bearer profile and region`, async () => { + const [a, b] = await seed(); + saveConfig(config()); + const refusalBody = reason === "OVERAGE_REQUEST_LIMIT_EXCEEDED" + ? { __type: "com.amazon.kiro.runtimeservice#ServiceQuotaExceededException", reason, + message: "You have reached the limit for overages." } : { reason }; + const observed: Array<{ url: string; auth: string | null; profile: string | null }> = []; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url.startsWith("https://runtime.") && url.endsWith(".kiro.dev/")) { + observed.push({ url, auth: new Headers(init?.headers).get("authorization"), + profile: new Headers(init?.headers).get("x-amzn-kiro-profile-arn") }); + if (observed.length === 1) return new Response(JSON.stringify(refusalBody), { status }); + return answer("served by b"); + } + return realFetch(input, init); + }) as typeof fetch; + const server = startServer(0); + try { + const response = await post(server, stream); + expect(response.status).toBe(200); + expect(await response.text()).toContain("served by b"); + expect(observed).toEqual([ + { url: "https://runtime.us-east-1.kiro.dev/", auth: "Bearer access-a", profile: aProfile }, + { url: "https://runtime.eu-west-1.kiro.dev/", auth: "Bearer access-b", profile: bProfile }, + ]); + expect(getAccountSet("kiro")!.activeAccountId).toBe(b!.id); + if (reason === "MONTHLY_REQUEST_COUNT" || reason === "OVERAGE_REQUEST_LIMIT_EXCEEDED") + expect(kiroAccountEvidence(a!).exhausted).toBe(true); + } finally { await server.stop(true); } + }); + } }