diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 48b71d4e255..ae36db1e029 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -410,6 +410,7 @@ Providers can expose a built-in shorthand, such as `agy` for `google-antigravity | — | — | Antigravity account quota probes (`retrieveUserQuota` and `retrieveUserQuotaSummary`) always go to Google's own Cloud Code host through the pinned outbound transport, regardless of a configured `baseUrl`; the account bearer is never sent to an operator-configured endpoint and a redirect aborts the probe. Only the model-list fallback still honors `baseUrl`. | | — | — | If Antigravity quota summary returns 403 for a valid OAuth account, OpenCodex retries that endpoint once with the legacy `antigravity/1.0` User-Agent and the same token and project. A 401 is not retried. Inference and model discovery retain the IDE User-Agent. | | — | — | Antigravity quota shows Gemini 4 Argon as separate `Argon` / `Argon (Weekly)` windows next to Gem and Cla. They are display-only: account ranking, quota exhaustion, reset ordering and reset notifications keep using the Gemini and Claude windows, and Argon requests are still judged by the Gemini window. | +| — | — | Refreshing Antigravity discovery unions usable accounts' enabled models and retains each account's own roster for six hours. With account failover active, a CCA 404 tries another eligible account only when its complete original error is at most 4 KiB and contains a model ResourceInfo whose resourceName exactly matches `models/`. Message-only errors, other resources and oversized, incomplete or erroring bodies keep the original HTTP status without a model denial or rotation. A confirmed denial lasts up to 24 hours, until positive discovery started after that request or credential replacement; older discovery cannot clear it or replace a newer positive roster. Exhaustion names the model and distinguishes unavailable accounts from credential, admission, or send-budget failure. No paid/trial classification is inferred. | | `location?` | `string` | Vertex location. API-key project endpoints default to `global` and ignore `GOOGLE_CLOUD_LOCATION`; ADC requires this field or the `GOOGLE_CLOUD_LOCATION` fallback. Gemini 4 Argon supports `global`, `us`, and `eu`. | | `mcpServers?` | `Record` | Cursor only: stdio or Streamable HTTP MCP servers. | | `desktopExecutor?` | `DesktopExecutorConfig` | Cursor only: external computer-use and record-screen commands. | diff --git a/scripts/test-layout/layout-agent-assertion.json b/scripts/test-layout/layout-agent-assertion.json index 6504b6670f2..203c673e19b 100644 --- a/scripts/test-layout/layout-agent-assertion.json +++ b/scripts/test-layout/layout-agent-assertion.json @@ -3,5 +3,7 @@ "agent-assertion-auth.test.ts": "server", "agent-assertion-routes.test.ts": "responses", "authorization-scheme.test.ts": "lib", - "server-antigravity-model-roster-routing.test.ts": "server" + "server-antigravity-model-roster-routing.test.ts": "server", + "antigravity-routing-carry.test.ts": "adapters/google", + "server-antigravity-model-denial-recovery.test.ts": "server" } diff --git a/src/adapters/antigravity-refusal.ts b/src/adapters/antigravity-refusal.ts index 6f0fed5571c..3aa8ba0abe9 100644 --- a/src/adapters/antigravity-refusal.ts +++ b/src/adapters/antigravity-refusal.ts @@ -1,5 +1,34 @@ /** Account-scoped Antigravity refusals. Unknown data never convicts an account. */ import { BOUNDED_BODY_MAX_BYTES } from "../lib/bounded-body"; +import { registerProviderResponseProvenance } from "../providers/request-pacing"; +import { parseUpstreamJsonPayload } from "./upstream-http-error"; + +/** Trusted only at the original Google body boundary; never serialized or inferred from display text. */ +const modelNotFoundResponses = new WeakMap(); +registerProviderResponseProvenance(modelNotFoundResponses); +export const hasAntigravityModelNotFoundProvenance = (response: Response): boolean => modelNotFoundResponses.has(response); + +/** The caller supplies a complete original body, bounded to 4 KiB before normalization. */ +export function markAntigravityModelNotFound(response: Response, requestBody: string, payloadText: string): void { + if (response.status !== 404 || Buffer.byteLength(payloadText, "utf8") > 4096) return; + const request = parseUpstreamJsonPayload(requestBody) as { model?: unknown } | undefined; + const payload = parseUpstreamJsonPayload(payloadText) as { error?: { + code?: unknown; status?: unknown; message?: unknown; details?: unknown; + } } | undefined; + const model = request?.model; + const error = payload?.error; + if (typeof model !== "string" || !model || !error || error.status !== "NOT_FOUND" + || (error.code !== undefined && error.code !== 404)) return; + const details = Array.isArray(error.details) ? error.details : []; + const resourceInfo = details.filter((detail): detail is Record => + detail !== null && typeof detail === "object" && !Array.isArray(detail) + && detail["@type"] === "type.googleapis.com/google.rpc.ResourceInfo"); + // No observed model-specific message form is known: prose alone cannot identify the missing resource. + if (!resourceInfo.length || resourceInfo.some(detail => detail.resourceName !== `models/${model}` + || (detail.resourceType !== undefined && (typeof detail.resourceType !== "string" + || !/(?:^|[./])models?$/i.test(detail.resourceType))))) return; + modelNotFoundResponses.set(response, true); +} export type AntigravityRefusalKind = "verify_account" | "other"; export interface AntigravityRefusal { kind: AntigravityRefusalKind } diff --git a/src/adapters/google-http.ts b/src/adapters/google-http.ts index cee4accea22..cbe371578ae 100644 --- a/src/adapters/google-http.ts +++ b/src/adapters/google-http.ts @@ -9,6 +9,7 @@ import { repairGoogleInvalidRequestBodyWithReport } from "./google-wire-compiler import type { GoogleToolSchemaPolicy, GoogleToolSchemaProfile } from "./google-tool-schema"; import { normalizeUpstreamHttpErrorResponse, readDisplaySafeErrorPayloadText } from "./upstream-http-error"; import { readBoundedResponseBody } from "../lib/bounded-body"; +import { markAntigravityModelNotFound } from "./antigravity-refusal"; import { abortError, cancelResponseBodyBestEffort, @@ -31,19 +32,27 @@ export interface GoogleRetryOptions { toolSchemaPolicy?: GoogleToolSchemaPolicy; } -async function normalizeFinalGoogleError(label: string, res: Response, signal?: AbortSignal): Promise { - if (label === "Antigravity" && res.status === 403) { - const body = await readBoundedResponseBody(res, { - maxBytes: 4096, totalTimeoutMs: 2000, firstByteTimeoutMs: 2000, - inactivityTimeoutMs: 2000, signal, - }); +async function normalizeFinalGoogleError(label: string, res: Response, request: AdapterRequest, signal?: AbortSignal): Promise { + if (label === "Antigravity" && (res.status === 403 || res.status === 404)) { + let payloadText = ""; + try { + const body = await readBoundedResponseBody(res, { + maxBytes: 4096, totalTimeoutMs: 2000, firstByteTimeoutMs: 2000, + inactivityTimeoutMs: 2000, signal, + }); + if (body.displaySafe && !body.truncated && !body.timedOut && !body.oversized) payloadText = body.text; + } catch (error) { + if (signal?.aborted) throw error; + } const headers = new Headers(res.headers); headers.delete("content-encoding"); headers.delete("content-length"); - return inheritAccountAutoPauseResponse(res, new Response(safeGoogleHttpErrorMessage(label, res.status, - body.displaySafe && !body.truncated ? body.text : ""), { + const normalized = inheritAccountAutoPauseResponse(res, new Response(safeGoogleHttpErrorMessage(label, res.status, + payloadText), { status: res.status, statusText: res.statusText, headers, })); + if (!signal?.aborted) markAntigravityModelNotFound(normalized, request.body, payloadText); + return normalized; } return normalizeUpstreamHttpErrorResponse(res, { signal, @@ -110,7 +119,7 @@ export async function fetchGoogleWithRetry( changedSendAllowed, })); if (!changedSendAllowed) { - return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, ctx.abortSignal); + return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, activeRequest, ctx.abortSignal); } } compatibilityReplayUsed = true; @@ -122,7 +131,7 @@ export async function fetchGoogleWithRetry( } } if (!retryableGoogleStatus(res.status) || attempt === TRANSIENT_RETRY_MAX_ATTEMPTS - 1) { - return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, ctx.abortSignal); + return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, activeRequest, ctx.abortSignal); } // A 429 may be a transient rate limit (retry) or hard quota exhaustion (do NOT retry — // it won't recover for hours and burns retries). Peek the body to tell them apart. @@ -147,7 +156,7 @@ export async function fetchGoogleWithRetry( retryAfterIsLowerBound: true, }); if (delayMs > RETRY_AFTER_CEILING_MS) { - return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, ctx.abortSignal); + return ctx.returnRawErrors ? res : normalizeFinalGoogleError(label, res, activeRequest, ctx.abortSignal); } pendingResponse = res; recovery = res.status === 429 ? "rate-limit-429" : "transient-5xx"; @@ -161,7 +170,7 @@ export async function fetchGoogleWithRetry( // upstream answer — but it used to leave the log indistinguishable from a request // where no retry was ever eligible (#5044). ctx.onRecoveryWithheld?.({ reason: "retry-send-budget" }); - return ctx.returnRawErrors ? pendingResponse : normalizeFinalGoogleError(label, pendingResponse, ctx.abortSignal); + return ctx.returnRawErrors ? pendingResponse : normalizeFinalGoogleError(label, pendingResponse, activeRequest, ctx.abortSignal); } throw err; } diff --git a/src/codex/catalog/antigravity-discovery.ts b/src/codex/catalog/antigravity-discovery.ts new file mode 100644 index 00000000000..97c78fe9e02 --- /dev/null +++ b/src/codex/catalog/antigravity-discovery.ts @@ -0,0 +1,59 @@ +import { getConfigDir } from "../../config/paths"; +import { getValidAccessSnapshotForAccount, type OAuthAccessSnapshot } from "../../oauth"; +import { credentialGeneration, getAccountCredentialWithStatus, getAccountSet } from "../../oauth/store"; +import { providerOutboundPost, providerRedirectError } from "../../lib/provider-outbound"; +import { isRegistryModelDiscoveryUrl, readBoundedDiscoveryJson } from "../../providers/model-discovery"; +import { parseAntigravityAvailableModels, type AntigravityAvailableModel } from "../../providers/antigravity-models"; +import { captureAntigravityModelObservation, recordAntigravityAccountRoster } from "../../providers/antigravity-account-roster"; +import type { OcxProviderConfig } from "../../types"; +import { materializeCapturedHeaders, type CapturedModelsRequest } from "./gather-capture"; + +/** A sibling contributes only while the exact snapshot sent is still usable. */ +export async function discoverAntigravitySiblings( + name: string, provider: OcxProviderConfig, activeAccountId: string, + request: CapturedModelsRequest, maxBytes: number, maxModels: number, + root: string, +): Promise<() => AntigravityAvailableModel[]> { + if (root !== getConfigDir()) return () => []; + const results: Array<{ + snapshot: OAuthAccessSnapshot; models: AntigravityAvailableModel[]; + observation: ReturnType; + }> = []; + const current = (snapshot: OAuthAccessSnapshot): boolean => { + if (getConfigDir() !== root) return false; + const row = getAccountCredentialWithStatus(name, snapshot.accountId); + return !!row && !row.paused && !row.needsReauth && row.credential.expires > Date.now() + && credentialGeneration(row.credential) === snapshot.generation + && row.credential.access === snapshot.accessToken; + }; + // Bound concurrency and preserve stable account order; a failed sibling is fail-soft. + for (const sibling of getAccountSet(name)?.accounts ?? []) { + if (sibling.id === activeAccountId || sibling.paused || sibling.needsReauth) continue; + try { + if (getConfigDir() !== root) break; + const snapshot = await getValidAccessSnapshotForAccount(name, sibling.id, { requireUsableAccount: true }); + const project = provider.project ?? snapshot.projectId; + if (!project || !current(snapshot)) continue; + const observation = captureAntigravityModelObservation(); + const response = await providerOutboundPost(name, provider, request.url, { + headers: materializeCapturedHeaders(request, snapshot.accessToken), + body: JSON.stringify({ project }), signal: AbortSignal.timeout(8000), + }, { isCanonicalUrl: isRegistryModelDiscoveryUrl, beforeSend: () => current(snapshot) }); + if (!response.ok || await providerRedirectError(response, request.url)) continue; + const bounded = await readBoundedDiscoveryJson(response, maxBytes); + if (!bounded.ok) continue; + const models = parseAntigravityAvailableModels(bounded.value, maxModels); + if (models && current(snapshot)) results.push({ snapshot, models, observation }); + } catch { /* A sibling cannot invalidate successful active-account discovery. */ } + } + // Called synchronously with the catalog publication, after all outbound awaits. + return () => { + const union: AntigravityAvailableModel[] = []; + for (const { snapshot, models, observation } of results) { + if (!current(snapshot)) continue; + recordAntigravityAccountRoster(name, snapshot.accountId, models, Date.now(), observation); + union.push(...models); + } + return union; + }; +} diff --git a/src/codex/catalog/provider-models.ts b/src/codex/catalog/provider-models.ts index 9af396a13ae..04238755718 100644 --- a/src/codex/catalog/provider-models.ts +++ b/src/codex/catalog/provider-models.ts @@ -53,7 +53,8 @@ import { import type { FastPolicyAuthority } from "../../providers/fastwire"; import { effectiveGoogleMode, getProviderRegistryEntry, providerMatchesRegistryTransport, registryEntryForProviderDestination } from "../../providers/registry"; import { parseAntigravityAvailableModels, registerAntigravityDiscoveredWireModels } from "../../providers/antigravity-models"; -import { recordAntigravityAccountRoster } from "../../providers/antigravity-account-roster"; +import { captureAntigravityModelObservation, recordAntigravityAccountRoster } from "../../providers/antigravity-account-roster"; +import { discoverAntigravitySiblings } from "./antigravity-discovery"; import { applyProviderContextCap, providerContextCap, resolveUnknownRoutedContextWindow } from "../../providers/context-cap"; import { clampAutoCompactTokenLimit } from "../../providers/auto-compact-budget"; import { effectiveModelAliases } from "../../providers/default-aliases"; @@ -729,6 +730,7 @@ export async function fetchProviderModelsWithAuth( // fetch a different URL and keep the rejection. if (!maySendAnthropicDiscovery()) return observed(withConfiguredRetention(failedDiscoveryConfigured), "degraded"); const outboundDependencies = { isCanonicalUrl: isRegistryModelDiscoveryUrl, beforeSend: maySendAnthropicDiscovery }; + const antigravityObservation = cloudCodeAssist ? captureAntigravityModelObservation() : undefined; const res = request.method === "POST" ? await providerOutboundPost(name, prov, url, { headers, @@ -790,7 +792,21 @@ export async function fetchProviderModelsWithAuth( return observed(models, "degraded"); } if (antigravity) { - const upstream: CatalogModel[] = antigravity.map(model => ({ + if (getConfigDir() !== metadataConfigDir) return observed(withConfiguredRetention(configured), "degraded"); + const admitSiblings = resolveAuth.kind === "refreshing" && name === "google-antigravity" + && prov.authMode === "oauth" && auth.oauthAccountId + ? await discoverAntigravitySiblings(name, prov, auth.oauthAccountId, request, discovery.maxResponseBytes, discovery.maxModels, metadataConfigDir) + : () => []; + if (!isCurrentCacheGeneration() || getConfigDir() !== metadataConfigDir) return observed(withConfiguredRetention(configured), "degraded"); + const activeAccount = auth.oauthAccountId ? getAccountCredentialWithStatus(name, auth.oauthAccountId) : null; + if (auth.oauthAccountId && (!activeAccount || activeAccount.paused || activeAccount.needsReauth + || activeAccount.credential.expires <= Date.now() || activeAccount.credential.access !== apiKey + || credentialGeneration(activeAccount.credential) !== auth.oauthGeneration)) + return observed(withConfiguredRetention(configured), "degraded"); + const byId = new Map(antigravity.map(model => [model.id, model])); + for (const model of admitSiblings()) if (!byId.has(model.id)) byId.set(model.id, model); + const merged = [...byId.values()]; + const upstream: CatalogModel[] = merged.map(model => ({ id: model.id, provider: name, // Only an exact discovered wire map proves a new model's selectable effort ladder. @@ -810,13 +826,13 @@ export async function fetchProviderModelsWithAuth( } // Save exact family routes before publishing synthetic IDs. These synchronous writes // cannot interleave with a credential-generation change; failure keeps the prior cache. - registerAntigravityDiscoveredWireModels(prov.baseUrl, antigravity, { + registerAntigravityDiscoveredWireModels(prov.baseUrl, merged, { provider: name, cacheGeneration, }); // Entitlement is per account (Claude 4.6 vs 5.5, Argon): remember what THIS account // lists so account selection can prefer one that serves the requested model. - if (auth.oauthAccountId) recordAntigravityAccountRoster(name, auth.oauthAccountId, antigravity); + if (auth.oauthAccountId) recordAntigravityAccountRoster(name, auth.oauthAccountId, antigravity, Date.now(), antigravityObservation); if (!setCached(name, forCache, Date.now(), cacheGeneration, catalogAuthority)) { return observed(withConfiguredRetention(configured), "degraded"); } diff --git a/src/oauth/generic-account-failover.ts b/src/oauth/generic-account-failover.ts index 30fa971eef0..a223f433e45 100644 --- a/src/oauth/generic-account-failover.ts +++ b/src/oauth/generic-account-failover.ts @@ -309,7 +309,7 @@ export function antigravitySelectedAccountUnlistedMessage(model: string): string * * `plannedAccountId` is the account the request would otherwise be sent with. It is kept unless * its own fresh roster omits the model: absence of a roster is unknown, never a denial, because - * rosters live in memory, are refreshed only for the account active during discovery, and expire + * rosters live in memory, are refreshed for usable accounts during discovery, and expire * after six hours, so a restart must not take every account out of service. * * A planned account with a proven omission is replaced by a usable sibling (not paused, not @@ -338,10 +338,36 @@ export function routeAntigravityModel( } const ready = new Set(eligibleFailoverAccounts(providerName, now, classifyModelFamilyForQuota(providerName, model))); const rank = (id: string) => (ready.has(id) ? 0 : 2) + (accountListsModel(providerName, id, model) === true ? 0 : 1); - const best = serving.reduce((left, right) => rank(right) < rank(left) ? right : left); + const tier = Math.min(...serving.map(rank)); + const best = rankAntigravityModelAccounts(config, serving.filter(id => rank(id) === tier), model, now)[0]!; return { kind: "reroute", accountId: best }; } +/** Strategy is applied only after readiness and model-support precedence. */ +function rankAntigravityModelAccounts(config: OcxConfig, ids: string[], model: string, now: number): string[] { + const provider = "google-antigravity"; + const rows = new Map((getAccountSet(provider)?.accounts ?? []).map(row => [row.id, row])); + const strategy = activeGenericStrategy(config, provider); + if (strategy === "round-robin") { + const best = peekRoundRobinAccount(genericPoolKey(provider), ids, genericStickyLimit(config, provider)); + return best ? [best, ...ids.filter(id => id !== best)] : ids; + } + if (strategy === "fill-first") return stableGenericRoster(provider).filter(id => ids.includes(id)); + if (strategy === "reset-first") return rankAccountsByResetFirst(provider, ids, model, now, rows); + return rankAccountsByHeadroom(provider, ids, model, rows); +} + +/** Reactive model refusal never changes cooldown or account-wide authentication state. */ +export function rotateAntigravityAccountOnModelNotFound( + config: OcxConfig, failedAccountId: string, model: string, attempted: ReadonlySet, now = Date.now(), +): string | null { + const candidates = preferKiroModelSupport("google-antigravity", + eligibleFailoverAccounts("google-antigravity", now, classifyModelFamilyForQuota("google-antigravity", model)) + .filter(id => id !== failedAccountId && !attempted.has(id)), model, config); + const positive = candidates.filter(id => accountListsModel("google-antigravity", id, model) === true); + return rankAntigravityModelAccounts(config, positive.length ? positive : candidates, model, now)[0] ?? null; +} + /** * Client-facing refusal when failover is on and a sibling may serve the model, but the request * still resolved to an omitting account (the sibling could not be resolved or admitted). @@ -757,6 +783,10 @@ export function preferredInitialAccount( const activeLacks = (providerName === "kiro" || providerName === "google-antigravity") && !!active && !!requestedModelId && accountListsModel(providerName, active, requestedModelId) === false; + if (providerName === "google-antigravity" && activeLacks) { + const route = routeAntigravityModel(config, providerName, active, requestedModelId, now); + return route.kind === "reroute" ? route.accountId : null; + } const listingSibling = activeLacks ? modelEligible.find(id => id !== active && (providerName === "kiro" ? kiroModelListingHasRoom(config, id, requestedModelId!) diff --git a/src/providers/antigravity-account-roster.ts b/src/providers/antigravity-account-roster.ts index 051798f3aa9..baa1c782b58 100644 --- a/src/providers/antigravity-account-roster.ts +++ b/src/providers/antigravity-account-roster.ts @@ -1,4 +1,5 @@ import { getConfigDir } from "../config/paths"; +import { credentialGeneration, getAccountCredentialWithStatus } from "../oauth/store"; import { ANTIGRAVITY_MODEL_ALIASES, canonicalAntigravityUsageModel, type AntigravityAvailableModel } from "./antigravity-models"; /** @@ -12,8 +13,8 @@ import { ANTIGRAVITY_MODEL_ALIASES, canonicalAntigravityUsageModel, type Antigra * needing a directly billed, non-trial Pro or Ultra plan and rolls it out gradually, so the static * lineup is a superset and only discovery says what one account can actually call. * - * The catalog already exposes only the ACTIVE account's discovered rows (an authoritative live - * roster drops static rows it did not return). This store keeps every account's last roster so + * Refreshing discovery exposes the union of usable accounts' discovered rows. This store keeps + * each account's own last roster and credential generation so * that account selection can prefer an account that lists the requested model, instead of * landing a 5.5 request on a 4.6-only account. Absence of a roster is "unknown", never "no": * an account that was never active during discovery stays fully usable. @@ -21,14 +22,57 @@ import { ANTIGRAVITY_MODEL_ALIASES, canonicalAntigravityUsageModel, type Antigra /** A roster older than this is treated as unknown; entitlements change with plan and rollout. */ export const ANTIGRAVITY_ACCOUNT_ROSTER_TTL_MS = 6 * 60 * 60 * 1000; +export const ANTIGRAVITY_MODEL_DENIAL_TTL_MS = 24 * 60 * 60 * 1000; + +/** Logical ordering survives observations completed in the same millisecond. */ +let observationOrder = 0; +export interface AntigravityModelObservation { readonly configDir: string; readonly order: number } +export function captureAntigravityModelObservation(): AntigravityModelObservation { + return { configDir: getConfigDir(), order: ++observationOrder }; +} +function accountGeneration(provider: string, accountId: string): string | undefined { + const row = getAccountCredentialWithStatus(provider, accountId); + return row ? credentialGeneration(row.credential) : undefined; +} +function denialModelId(modelId: string): string { + const id = modelId.trim(); + return canonicalAntigravityUsageModel(Object.hasOwn(ANTIGRAVITY_MODEL_ALIASES, id) ? ANTIGRAVITY_MODEL_ALIASES[id]! : id); +} interface AccountRoster { readonly ids: ReadonlySet; readonly modelIds: readonly string[]; readonly observedAt: number; + readonly generation?: string; + readonly order: number; } const rosters = new Map(); +const modelDenials = new Map>(); + +/** Admit only evidence from the credential actually sent, in the captured configuration home. */ +export function recordAntigravityModelDenial( + provider: string, accountId: string, modelId: string, + sent: AntigravityModelObservation & { readonly generation: string }, now = Date.now(), +): boolean { + if (getConfigDir() !== sent.configDir || accountGeneration(provider, accountId) !== sent.generation) return false; + const key = rosterKey(provider, accountId); + const roster = rosters.get(key); + const id = denialModelId(modelId); + if (!id) return false; + if (roster?.generation === sent.generation && roster.order > sent.order + && (roster.ids.has(id) || roster.ids.has(modelId.trim()))) return false; + let denials = modelDenials.get(key); + if (!denials) { + if (modelDenials.size >= 2000) modelDenials.delete(modelDenials.keys().next().value!); + modelDenials.set(key, denials = new Map()); + } + const previous = denials.get(id); + if (previous && previous.order > sent.order) return false; + if (denials.size >= 2000 && !denials.has(id)) denials.delete(denials.keys().next().value!); + denials.set(id, { generation: sent.generation, order: sent.order, deniedAt: now }); + return true; +} function rosterKey(providerName: string, accountId: string): string { return `${getConfigDir()}\0${providerName}\0${accountId}`; @@ -40,24 +84,38 @@ export function recordAntigravityAccountRoster( accountId: string, models: readonly AntigravityAvailableModel[], now = Date.now(), + observation = captureAntigravityModelObservation(), ): void { - if (!providerName || !accountId) return; + if (!providerName || !accountId || observation.configDir !== getConfigDir()) return; + const key = rosterKey(providerName, accountId); + const generation = accountGeneration(providerName, accountId); + const previous = rosters.get(key); + if (previous && previous.generation === generation && previous.order > observation.order) return; const ids = new Set(); for (const model of models) { ids.add(model.id); ids.add(model.wireModelId); for (const wire of Object.values(model.effortWireModelIds ?? {})) if (wire) ids.add(wire); } - rosters.set(rosterKey(providerName, accountId), { + rosters.set(key, { ids, modelIds: models.map(model => model.id), observedAt: now, + generation, + order: observation.order, }); + const denials = modelDenials.get(key); + for (const id of ids) { + const key = denialModelId(id); + const denial = denials?.get(key); + if (denial && denial.order <= observation.order) denials!.delete(key); + } } function freshRoster(providerName: string, accountId: string, now: number): AccountRoster | undefined { const roster = rosters.get(rosterKey(providerName, accountId)); - if (!roster || now - roster.observedAt > ANTIGRAVITY_ACCOUNT_ROSTER_TTL_MS) return undefined; + if (!roster || now - roster.observedAt > ANTIGRAVITY_ACCOUNT_ROSTER_TTL_MS + || roster.generation !== accountGeneration(providerName, accountId)) return undefined; return roster; } @@ -81,10 +139,16 @@ export function antigravityAccountSupportsModel( modelId: string, now = Date.now(), ): boolean | undefined { - const roster = freshRoster(providerName, accountId, now); - if (!roster) return undefined; const id = modelId.trim(); if (!id) return undefined; + const key = rosterKey(providerName, accountId); + const canonicalId = denialModelId(id); + const denial = modelDenials.get(key)?.get(canonicalId); + const roster = freshRoster(providerName, accountId, now); + if (denial && denial.generation === accountGeneration(providerName, accountId) + && now - denial.deniedAt <= ANTIGRAVITY_MODEL_DENIAL_TTL_MS + && (!roster || denial.order >= roster.order)) return false; + if (!roster) return undefined; if (roster.ids.has(id)) return true; const canonical = canonicalAntigravityUsageModel(id); if (roster.ids.has(canonical)) return true; @@ -96,7 +160,9 @@ export function antigravityAccountSupportsModel( export function clearAntigravityAccountRosters(providerName?: string, accountId?: string): void { if (providerName && accountId) { rosters.delete(rosterKey(providerName, accountId)); + modelDenials.delete(rosterKey(providerName, accountId)); return; } rosters.clear(); + modelDenials.clear(); } diff --git a/src/providers/antigravity-models.ts b/src/providers/antigravity-models.ts index 901f07560ec..73ea76c5967 100644 --- a/src/providers/antigravity-models.ts +++ b/src/providers/antigravity-models.ts @@ -608,6 +608,9 @@ export function parseAntigravityAvailableModels( if (!models || !sorts) return null; const ids: string[] = []; + // Disabled and duplicate ids cannot consume the enabled-model budget, but raw work is bounded. + const rawLimit = MODEL_DISCOVERY_MAX_MODELS * 4; + let rawCount = 0; for (const sort of sorts) { const groups = antigravityRecord(sort)?.groups; if (!Array.isArray(groups)) return null; @@ -618,7 +621,7 @@ export function parseAntigravityAvailableModels( if (!isValidModelDiscoveryModelId(id) || !Object.hasOwn(models, id) || !antigravityRecord(models[id]) - || ids.length >= limit) return null; + || ++rawCount > rawLimit) return null; ids.push(id); } } @@ -630,7 +633,7 @@ export function parseAntigravityAvailableModels( && Object.hasOwn(models, "gemini-3.1-flash-image") && antigravityRecord(models["gemini-3.1-flash-image"]) && !ids.includes("gemini-3.1-flash-image")) { - if (ids.length >= limit) return null; + if (++rawCount > rawLimit) return null; ids.push("gemini-3.1-flash-image"); } // Newer CCA responses identify tiered Flash models through this index instead of @@ -642,7 +645,7 @@ export function parseAntigravityAvailableModels( if (!isValidModelDiscoveryModelId(id) || !Object.hasOwn(models, id) || !antigravityRecord(models[id]) - || ids.length >= limit) return null; + || ++rawCount > rawLimit) return null; const baseId = id.endsWith("-tiered") ? id.slice(0, -"-tiered".length) : id; if (ids.some(agentId => agentId === id @@ -656,7 +659,8 @@ export function parseAntigravityAvailableModels( const available = new Map>(); for (const wireId of ids) { const info = antigravityRecord(models[wireId]); - if (!info || available.has(wireId)) continue; + if (!info || info.disabled === true || wireId.endsWith("-disabled") || available.has(wireId)) continue; + if (available.size >= limit) return null; // Compatibility aliases are deliberately routed to NEWER wire ids for saved // selections. CCA keeps serving retired generations (3.5/3.6 Flash tiers) in its // agent list long after they stop being the model anyone should pick, so admitting diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 65e75bc27ea..bed97287048 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -116,6 +116,7 @@ import { isCanonicalOpenAiForwardProvider } from "../../providers/openai-tiers"; import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; import { chargeWorkflowSends } from "../../lib/workflow-budget"; import { isAntigravityValidationRefusal } from "./antigravity-validation-refusal"; +import { createAntigravityModelDenialRecovery, recoverAntigravityAccount } from "./antigravity-model-denial-recovery"; import { unboundPoolSpendRefusalResponse } from "../workflow-refusal"; import { AdmissionModelDeniedError, admissionModelDeniedResponse } from "../admission-model-scope"; @@ -270,6 +271,7 @@ export async function prepareAdapterExchange( transportState.activeAdapter = transportState.adapter; const antigravityPoolActivated = route.providerName === "google-antigravity" && isGenericOAuthFailoverEnabled(config, route.providerName); + const antigravityDenial = createAntigravityModelDenialRecovery(); // One immutable, body-safe outbound request per same-target sequence (URL, serialized body, // auth headers, generated compat headers). Same-target 429 replays reuse it verbatim; the @@ -416,7 +418,7 @@ export async function prepareAdapterExchange( abortSignal: upstream.signal, timeoutMs: connectMs, sendBudget: adapterDispatchBudget, - onPhysicalSend: send => noteAdapterPhysicalSend(inputTokenEstimate, send), + onPhysicalSend: send => { antigravityDenial.beforeSend(); noteAdapterPhysicalSend(inputTokenEstimate, send); }, onRecoveryWithheld: noteAdapterRecoveryWithheld, stream: parsed.stream, executor: providerFetch(route.provider, options.codexWsRuntimeIdentity, { @@ -717,7 +719,7 @@ export async function prepareAdapterExchange( sendBudget: adapterDispatchBudget, onPhysicalSend: send => { if (preserveFailureResponse) replacementAdmitted = true; - noteAdapterPhysicalSend(retryEstimate, send); + antigravityDenial.beforeSend(); noteAdapterPhysicalSend(retryEstimate, send); chargeFastDowngradeWorkflowSend(); }, onRecoveryWithheld: noteAdapterRecoveryWithheld, @@ -883,23 +885,23 @@ export async function prepareAdapterExchange( const rotateAntigravityAuth = async ( failedResponse: Response, recovery: AttemptRecoveryKind, + modelAccountId?: string, ): Promise => { const sent = transportState.replayOAuthCredentialSnapshot ?? transportState.sentOAuthSnapshot; if (!antigravityPoolActivated || !sent || transportState.genericFailovers >= transportState.genericFailoverLimit) return null; - const nextId = rotateAntigravityAccountOnAuthRefusal( + const nextId = modelAccountId ?? rotateAntigravityAccountOnAuthRefusal( antigravityPoolActivated, sent.accountId, sent.generation, route.modelId, ); if (!nextId) return null; const adapterOwnsDispatch = transportState.activeAdapter.fetchResponse !== undefined; - const hop = reserveCredentialHop("auth-recovery", - `${route.providerName}|${route.modelId}|antigravity-auth`, - !adapterOwnsDispatch && genericRefetchCountsSends()); - if (!hop.allowed) return null; - try { - const snapshot = await failoverAccountSnapshot(route.providerName, nextId); - const admitted = await applyFailoverSnapshot(snapshot); - if (admitted?.accountId !== nextId) return null; + return recoverAntigravityAccount({ nextId, failedResponse, adapterOwnsDispatch, + reserve: () => reserveCredentialHop("auth-recovery", `${route.providerName}|${route.modelId}|antigravity-auth`, + !adapterOwnsDispatch && genericRefetchCountsSends()), + applyFailoverSnapshot, setPermit: permit => { sendBudgetState.pendingHopPermit = permit; }, + complete: () => { transportState.genericFailovers += 1; }, + refetch: onDispatch => rebuildAndRefetch(recovery, onDispatch, failedResponse), + prepare: () => { invalidateSameTargetRequest(); transportState.activeAdapter = resolveSelectionAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy), @@ -913,23 +915,18 @@ export async function prepareAdapterExchange( transportState.activeAdapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); - sendBudgetState.pendingHopPermit = hop.permit; - const result = await rebuildAndRefetch(recovery, () => { - if (!adapterOwnsDispatch) hop.permit?.use(); - }, failedResponse); - if ("failed" in result) return result.failed === failedResponse ? null : result.failed; - transportState.genericFailovers += 1; - try { void failedResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } - return result; - } catch { - return null; - } finally { - sendBudgetState.pendingHopPermit = undefined; - hop.permit?.release(); - } + }, + }); }; // Keep recovery kinds in sync with the native Responses `passthroughRecovery:` loop above. recovery: for (;;) { + const modelRotation = await antigravityDenial.recover(upstreamResponse, + transportState.replayOAuthCredentialSnapshot ?? transportState.sentOAuthSnapshot, { + config, model: route.modelId, enabled: antigravityPoolActivated && transportState.activeAdapter.name === "google", + remaining: transportState.genericFailovers < transportState.genericFailoverLimit, signal: options.abortSignal, + rotate: id => rotateAntigravityAuth(upstreamResponse, "oauth-account-403", id), + }); + if (modelRotation) { upstreamResponse = modelRotation; continue recovery; } if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") { const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop, remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed), @@ -1934,7 +1931,7 @@ export async function prepareAdapterExchange( errorType: normalized.type, code: normalized.code, message: "Structured upstream failure observed before client formatting", }); - const message = normalized.cyberPolicy + const message = antigravityDenial.message(route.modelId, normalized.cyberPolicy ? normalized.message ?? (isCyberPolicyCode(normalized.code) ? CYBER_POLICY_FALLBACK_MESSAGE : normalized.safeText) : enrichOpenCodeZenUpstreamMessage( @@ -1953,7 +1950,7 @@ export async function prepareAdapterExchange( // never reach enrichOpenCodeZenUpstreamMessage here. supportsHttpSameKeyRetry: true, }, - ); + )); const retryAfter = normalized.cyberPolicy ? undefined : resolveClientRetryAfter({ diff --git a/src/server/responses/antigravity-model-denial-recovery.ts b/src/server/responses/antigravity-model-denial-recovery.ts new file mode 100644 index 00000000000..4cb6bf5a285 --- /dev/null +++ b/src/server/responses/antigravity-model-denial-recovery.ts @@ -0,0 +1,70 @@ +import { getConfigDir } from "../../config/paths"; +import type { OAuthAccessSnapshot } from "../../oauth"; +import { failoverAccountSnapshot, rotateAntigravityAccountOnModelNotFound } from "../../oauth/generic-account-failover"; +import { captureAntigravityModelObservation, recordAntigravityModelDenial } from "../../providers/antigravity-account-roster"; +import { isNonReplayableResponse } from "../../lib/upstream-retry"; +import type { OcxConfig } from "../../types"; +import type { ResponsesSendBudget } from "./request-send-budget"; +import { isAntigravityModelNotFoundRefusal } from "./antigravity-validation-refusal"; + +/** Common guarded credential-hop lifecycle for Antigravity auth and model recovery. */ +export async function recoverAntigravityAccount(options: { + nextId: string; failedResponse: Response; + reserve: () => ReturnType; + applyFailoverSnapshot: (snapshot: OAuthAccessSnapshot) => Promise; + prepare: () => void; + refetch: (onDispatch: () => void) => Promise; + adapterOwnsDispatch: boolean; + setPermit: (permit: ResponsesSendBudget["pendingHopPermit"]) => void; + complete: () => void; +}): Promise { + const hop = options.reserve(); + if (!hop.allowed) return null; + try { + const snapshot = await failoverAccountSnapshot("google-antigravity", options.nextId); + if ((await options.applyFailoverSnapshot(snapshot))?.accountId !== options.nextId) return null; + options.prepare(); + options.setPermit(hop.permit); + const result = await options.refetch(() => { if (!options.adapterOwnsDispatch) hop.permit?.use(); }); + if ("failed" in result) return result.failed === options.failedResponse ? null : result.failed; + options.complete(); + try { void options.failedResponse.body?.cancel().catch(() => {}); } catch { /* already closed */ } + return result; + } catch { return null; } + finally { options.setPermit(undefined); hop.permit?.release(); } +} + +/** Request-local ordering fence and attempted-account set; never infer exhaustion from a failed hop. */ +export function createAntigravityModelDenialRecovery() { + const root = getConfigDir(); + let observation = captureAntigravityModelObservation(); + const attempted = new Set(); + let exhaustion: "none" | "ineligible" | "recovery" = "none"; + return { + beforeSend: () => { observation = captureAntigravityModelObservation(); }, + async recover(response: Response, sent: Pick | undefined, context: { + config: OcxConfig; model: string | null | undefined; enabled: boolean; remaining: boolean; + signal?: AbortSignal; rotate: (id: string) => Promise; + }): Promise { + if (!context.enabled || !sent || !context.model || isNonReplayableResponse(response) + || !await isAntigravityModelNotFoundRefusal(response, context.signal)) return null; + // Capture BEFORE the bounded response read: another discovery can finish during it. + if (getConfigDir() !== root || !recordAntigravityModelDenial("google-antigravity", sent.accountId, + context.model, { ...observation, configDir: root, generation: sent.generation })) return null; + attempted.add(sent.accountId); + const next = rotateAntigravityAccountOnModelNotFound(context.config, sent.accountId, context.model, attempted); + if (!next) { exhaustion = "ineligible"; return null; } + if (!context.remaining) { exhaustion = "recovery"; return null; } + const result = await context.rotate(next); + exhaustion = result ? "none" : "recovery"; + return result; + }, + message(model: string | null | undefined, fallback: string): string { + return exhaustion === "ineligible" + ? `No eligible stored Google Antigravity account can serve the model "${model}". ${fallback}` + : exhaustion === "recovery" + ? `Google Antigravity could not recover the model "${model}": an eligible account could not be resolved, admitted, or sent within the request budget. ${fallback}` + : fallback; + }, + }; +} diff --git a/src/server/responses/antigravity-validation-refusal.ts b/src/server/responses/antigravity-validation-refusal.ts index 7c3512698a4..e91a5d8b6e7 100644 --- a/src/server/responses/antigravity-validation-refusal.ts +++ b/src/server/responses/antigravity-validation-refusal.ts @@ -1,5 +1,6 @@ import { ANTIGRAVITY_VALIDATION_REQUIRED_PREFIX, safeAntigravityHttpErrorMessage } from "../../adapters/google-errors"; import { readBoundedResponseBody } from "../../lib/bounded-body"; +import { hasAntigravityModelNotFoundProvenance } from "../../adapters/antigravity-refusal"; /** Match only the normalized Antigravity validation-refusal marker. */ export function hasAntigravityValidationRefusalMarker(text: string): boolean { @@ -32,3 +33,8 @@ async function inspectValidationRefusal(response: Response, signal: AbortSignal return false; } } + +/** Only original-body, model-specific Google adapter evidence qualifies for account rotation. */ +export async function isAntigravityModelNotFoundRefusal(response: Response, signal?: AbortSignal): Promise { + return response.status === 404 && !signal?.aborted && hasAntigravityModelNotFoundProvenance(response); +} diff --git a/structure/catalog.md b/structure/catalog.md index e0faab25764..31d969b3147 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -25,7 +25,7 @@ validation-pending. Import alone supplies no entitlement evidence for the model `src/server/models-capabilities.ts` preserves the legacy `/v1/models` negative reasoning signal for an explicit empty effort ladder when no reasoning, thinking-toggle or effort capability is declared. Explicit capability axes take precedence; a declared thinking toggle or reasoning support remains available with no effort rungs. This compatibility projection does not change Codex ladders, Claude thinking discovery, client exports or request wire policy. Vision-sidecar attachment metadata is projected after provider discovery/cache reads in `src/codex/catalog/routed-gather.ts`. The gather captures `visionSidecar.enabled !== false` before awaiting discovery, and `src/codex/catalog/gather-capture.ts` includes that effective flag in the flight key. Provider TTL/stale rows exclude synthetic sidecar image input; custom rows and synthesized combo members use the same captured switch. Explicit/native image modalities remain intact, and enabled advertisement retains the existing config-based consumer predicate without probing credential availability. An effective Vision switch change through `src/server/management/config-routes.ts` invokes the catalog-only management funnel after persistence to publish both catalog and cache; incomplete work is exposed as `catalogRefreshPending` without claiming client reload. Coverage: `tests/codex-integration/catalog-vision-sidecar-disabled.test.ts` and `tests/vision/sidecar-settings-vision-catalog.test.ts`. Custom-provider missing windows use [source-labelled vendor estimates](catalog-vendor-context.md) only when the provider opts in with `inferContextWindow: true`; these remain distinct from asserted routing capacity. Registry context seed capture preserves [configured window precedence](catalog-vendor-context.md): provider-wide limits bound seed defaults, explicit per-model windows win, and smaller observed limits remain authoritative. -Static policy and observed catalog evidence are separate authorities. Antigravity grouping and selection projection follow the [discovered effort-family contract](providers-and-adapters.md#antigravity-effort-families). +Static policy and observed catalog evidence are separate authorities. Antigravity grouping and selection projection follow the [discovered effort-family contract](providers-and-adapters.md#antigravity-effort-families). Refreshing Antigravity discovery unions enabled models from usable accounts through `src/codex/catalog/antigravity-discovery.ts`; final-send and publication checks retain each sibling's config-root and full credential snapshot authority. Each account publishes its own roster; active and sibling discovery capture observation order before sending, so delayed publication cannot erase a later inference denial. Active publication also rechecks the captured credential generation. An observed-only gather does not probe siblings. `src/providers/resolved-model-policy.ts` resolves and freezes only registry/operator static facts, hard wire pins, aliases, and explicit false/empty declarations. Discovery responses, generated metadata, cache freshness, availability, credentials, account state, quota and health never enter diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index 6dcb4795f22..805f8ba0104 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -581,7 +581,7 @@ Antigravity discovery in `src/providers/antigravity-models.ts` collapses complet Antigravity catalog rows carry the observed complete effort map from `src/providers/antigravity-models.ts` through the in-memory cache. `src/providers/antigravity-effort-families.ts` uses that evidence to group public rows and project suffix allowlists into the base selection, including both final catalog merge callers. Internal retained suffix rows remain available to combo resolution. Authoritative new-model reconciliation normalizes both the prior baseline and policy input before detecting arrivals: an existing suffix family is not a new model. All-disabled tiers transfer a base disable once; a known base keeps the operator's later choice. Genuinely new families still follow new-model policy. Saved defaults, combo targets, suffix disables and provider selections remain unchanged. A discovered ladder advertises only its wire-backed efforts unless explicitly overridden; unknown single-wire models gain no invented ladder. -Known Claude Sonnet and Opus 5.5 base and low/medium/high IDs share a deterministic usage identity in `src/providers/antigravity-models.ts`, independently of live discovery. This accounting map does not migrate saved routing selections; Claude 4.6 and unknown suffix identities remain unchanged. Antigravity model entitlement is per account: one account may list Claude 4.6, another only the Claude 5.5 suffix families, and Gemini 4 Argon only on Ultra/Enterprise. `ANTIGRAVITY_MODELS` is therefore a static superset (Gemini 3.8 Flash, Gemini 4 Argon, Claude Opus 5.5, Claude Sonnet 5.5), and `ANTIGRAVITY_KNOWN_PICKER_MODELS` keeps dropped bases known to routing, discovery collapse and usage identity. The catalog exposes only what the active account's authoritative discovery returns. `src/providers/antigravity-account-roster.ts` records each account's discovered roster (in memory, six-hour freshness, scoped to config home); `src/oauth/generic-account-failover.ts` gates Antigravity accounts by it: a listing account wins, an account without a fresh roster is unknown and stays usable, and an account whose fresh roster omits the model is never selected, so preference, 429 rotation and auth-refusal rotation answer no account instead of falling back to it (Kiro listings stay a preference). Before credential resolution, `routeAntigravityModel` lets `src/server/responses/request-transport.ts` replace an omitting planned account with a usable listing, then unknown, sibling when proactive preference is enabled; otherwise, and when no usable account may serve the model, the transport refuses locally with 403 `permission_denied` before any send. The shared admission step (`commitResolvedOAuthSelection`) re-checks every account it admits, so a resolution fallback, 401 recovery or failover rebind onto an omitting account is abandoned instead of sent. `tests/server/server-antigravity-model-roster-routing.test.ts` covers that boundary. Gemini 4 Argon requests and Antigravity model discovery send the Antigravity CLI User-Agent; every other model keeps the IDE User-Agent. +Known Claude Sonnet and Opus 5.5 base and low/medium/high IDs share a deterministic usage identity in `src/providers/antigravity-models.ts`, independently of live discovery. This accounting map does not migrate saved routing selections; Claude 4.6 and unknown suffix identities remain unchanged. Antigravity model entitlement is per account: one account may list Claude 4.6, another only the Claude 5.5 suffix families, and Gemini 4 Argon only on Ultra/Enterprise. `ANTIGRAVITY_MODELS` is therefore a static superset (Gemini 3.8 Flash, Gemini 4 Argon, Claude Opus 5.5, Claude Sonnet 5.5), and `ANTIGRAVITY_KNOWN_PICKER_MODELS` keeps dropped bases known to routing, discovery collapse and usage identity. Refreshing discovery exposes the union of usable accounts' enabled models. `src/codex/catalog/antigravity-discovery.ts` uses full account snapshots and checks config root, account presence, pause/reauth state and credential generation at final send and again before roster/union publication; failed siblings contribute nothing. Active and sibling discovery capture observation order before sending; delayed active publication retains newer inference denials and rechecks its captured credential generation. Roster publication rejects observations older than the stored same-generation observation before changing model IDs or clearing denials. Enabled entries alone consume the configured model limit, with a separate raw-entry bound. `src/providers/antigravity-account-roster.ts` records each account's discovered roster (in memory, six-hour freshness, scoped to config home and credential generation); `src/oauth/generic-account-failover.ts` gates Antigravity accounts by it: a listing account wins, an account without a fresh roster is unknown and stays usable, and an account whose fresh roster omits the model is never selected, so preference, 429 rotation and auth-refusal rotation answer no account instead of falling back to it (Kiro listings stay a preference). Before credential resolution, `routeAntigravityModel` lets `src/server/responses/request-transport.ts` replace an omitting planned account with a usable listing, then unknown, sibling when proactive preference is enabled; otherwise, and when no usable account may serve the model, the transport refuses locally with 403 `permission_denied` before any send. The shared admission step (`commitResolvedOAuthSelection`) re-checks every account it admits, so a resolution fallback, 401 recovery or failover rebind onto an omitting account is abandoned instead of sent. `tests/server/server-antigravity-model-roster-routing.test.ts` covers that boundary. Gemini 4 Argon requests and Antigravity model discovery send the Antigravity CLI User-Agent; every other model keeps the IDE User-Agent. `retrieveUserQuotaSummary` reports Gemini 4 Argon as its own group; `src/providers/quota/antigravity.ts` decides each bucket's family from its own id/name (Argon before Gemini, then Claude/3P/GPT), and only for an inconclusive bucket falls back to the group: Argon from the group display name alone (never its description), Gemini and Claude/3P/GPT from the existing display-name-plus-description rules. Argon buckets become `Argon`/`Argon (Weekly)` windows tagged with the canonical pool model `gemini-4-argon`, so they never fill or replace Gem/Cla in either group order and stay diagnostic like the model windows below; an Argon-only summary is a display report with no routing evidence, and the compact GUI summary and tray aggregation treat it like other diagnostic-only data. When the summary endpoint has no usable result, the same file preserves bounded per-model `quotaInfo` measurements from `fetchAvailableModels` alongside the existing first-seen Gem/Cla family summaries, where an Argon model never seeds Gem. Model windows carry the exact CCA wire ID and distinct labels of at most 128 characters for display and diagnostics; GUI summaries and native compact rows preserve the discriminator, retain details, and exclude model readings from headline and severity selection; account and provider routing ignore these rows and continue to use family summaries, and `src/quota/window-mapping.ts` keeps them out of quota-reset observation so one pool reset notifies once. A missing model row or failed probe is unknown evidence, not a model eligibility verdict. Per-account cache entries are bound to the current credential and project identity. Paused reads retain only current last-good measurements without probing; `src/providers/quota/account-row.ts` preserves private currentness callbacks through account-row projection until management publication. Account-row projection is loaded with `import()` after each account probe in `src/providers/quota.ts`; its cache dependencies do not create a static return edge into the quota facade. diff --git a/structure/providers/generic-account-pool.md b/structure/providers/generic-account-pool.md index 26846e9af1b..52905b19b79 100644 --- a/structure/providers/generic-account-pool.md +++ b/structure/providers/generic-account-pool.md @@ -26,6 +26,8 @@ are excluded. `tests/oauth/generic-session-affinity.test.ts` and ## Combo sibling probes +Antigravity model routing in `src/oauth/generic-account-failover.ts` ranks siblings by configured strategy or family headroom only within the best readiness/model-support tier. A known omission stays excluded, and unknown support stays usable. Reactive model-404 rotation excludes accounts already attempted in the request and prefers positive support before unknown support without changing account-wide auth or cooldown state. + `src/server/responses/core-combo.ts` asks `src/oauth/generic-account-failover.ts` whether a refused account has an eligible sibling before cooling the whole target. This probe records the refused account's cooldown but passes `commit: false`, so it only peeks at a round-robin pool. An unsent diff --git a/structure/transports/inventory.md b/structure/transports/inventory.md index 37699c0a913..c623590c06f 100644 --- a/structure/transports/inventory.md +++ b/structure/transports/inventory.md @@ -38,7 +38,7 @@ surface is listed here so a maintainer can find the owner without grepping: | AWS eventstream | `src/lib/eventstream-decoder.ts` | Kiro's prelude and message checksums use `Bun.hash.crc32` over the supplied byte view, with unsigned IEEE CRC-32 results and incremental seed support. Corrupt or truncated frames fail decoding. The streaming decoder rejects advertised sizes above 16 MiB per frame or 128 KiB per header block once the corresponding prelude bytes are available. `tests/responses/eventstream-decoder.test.ts` covers fixed wire bytes, offset views, seeded checksums, corruption, and fragmented input. | | Responses custom-tool preview | `src/bridge/sse.ts`, `src/server/responses-custom-tool-repair.ts`, `src/responses/progressive-freeform-input.ts`, `src/responses/freeform-wrapper-scan.ts` | Direct adapter events and routed function restoration share one progressive wrapper decoder over one bounded JSON classification of the prefix, so property order and escaped key spellings preview as the wrapper completion unwraps them. Fence-shaped `exec`/`apply_patch` prefixes stay held until authoritative completion normalization, while each caller retains its own patch-envelope and byte-budget policy. | | Meta Muse Responses tool names | `src/responses/muse-tool-name-alias.ts`, `src/adapters/openai-responses.ts` | `api.meta.ai` only: function names over 64 characters or containing characters outside `[a-zA-Z0-9_-]` become collision-safe wire aliases and are restored before the client sees them. | -| Google / Vertex / Antigravity | `src/adapters/google.ts`, `src/adapters/google-http.ts`, `src/adapters/google-wire-compiler.ts`, `src/adapters/google-tool-schema.ts`, `src/adapters/google-truncation.ts`, `src/adapters/google-errors.ts`, `src/adapters/google-antigravity-wire.ts`, `src/adapters/google-antigravity-replay.ts`, `src/adapters/google-wire-shape.ts` | Vertex and Antigravity install a Google-family `fetchResponse` and so own their retry policy, while AI Studio Gemini leaves it undefined and uses the default server fetch path. Its transient ladder uses the shared `TRANSIENT_RETRY_MAX_ATTEMPTS` constant, also used to fund each eligible generic OAuth account in the [request send budget](responses-spend.md#credential-hop-reservations). A `Retry-After` on a retryable 429/503 is the earliest resend: the two-second local maximum bounds only the wrapper's own jittered backoff, and an instruction past the shared `RETRY_AFTER_CEILING_MS` (60 seconds) ends with that upstream response, `Retry-After` intact, instead of an early resend. Antigravity 403 normalization reads at most 4096 bytes for at most two seconds and emits a validation marker only for a complete structured `VALIDATION_REQUIRED` reason; an unsafe or incomplete body falls back to status-only text. The Google-family wrapper reuses shared abort/deadline helpers, upstream error normalization, and policy-aware wire-body repair: strict initial schema loss sends nothing, while strict repair withholding returns the original 400 without a changed send. The final compiler produces the [content-free tool-schema loss contract](../providers/google.md#google-tool-schema-loss-reporting). `google-wire-shape.ts` remains diagnostic-only. | +| Google / Vertex / Antigravity | `src/adapters/google.ts`, `src/adapters/google-http.ts`, `src/adapters/google-wire-compiler.ts`, `src/adapters/google-tool-schema.ts`, `src/adapters/google-truncation.ts`, `src/adapters/google-errors.ts`, `src/adapters/google-antigravity-wire.ts`, `src/adapters/google-antigravity-replay.ts`, `src/adapters/google-wire-shape.ts` | Vertex and Antigravity install a Google-family `fetchResponse` and so own their retry policy, while AI Studio Gemini leaves it undefined and uses the default server fetch path. Its transient ladder uses the shared `TRANSIENT_RETRY_MAX_ATTEMPTS` constant, also used to fund each eligible generic OAuth account in the [request send budget](responses-spend.md#credential-hop-reservations). A `Retry-After` on a retryable 429/503 is the earliest resend: the two-second local maximum bounds only the wrapper's own jittered backoff, and an instruction past the shared `RETRY_AFTER_CEILING_MS` (60 seconds) ends with that upstream response, `Retry-After` intact, instead of an early resend. Antigravity 403/404 normalization reads at most 4096 bytes for at most two seconds and emits a validation marker only for a complete structured `VALIDATION_REQUIRED` reason; an unsafe, incomplete or erroring body falls back to status-only text with the original HTTP status; caller cancellation propagates. Model-404 denial provenance requires an exact model ResourceInfo under the [account refusal contract](responses-failover.md#account-refusal-and-rotation-boundaries). The Google-family wrapper reuses shared abort/deadline helpers, upstream error normalization, and policy-aware wire-body repair: strict initial schema loss sends nothing, while strict repair withholding returns the original 400 without a changed send. The final compiler produces the [content-free tool-schema loss contract](../providers/google.md#google-tool-schema-loss-reporting). `google-wire-shape.ts` remains diagnostic-only. | | Mimo Free | `src/adapters/mimo-free.ts` | Client identity and JWT handling are transport-local; the per-install client id lives in the opencodex state root. | | Anthropic image ingress | `src/adapters/anthropic-image-guard.ts`, `src/adapters/anthropic-image-normalize.ts`, `src/adapters/anthropic-image-codec.ts` | Oversized or unsupported images are normalized or rejected before reaching upstream. An image's ladder position is pinned to its own fixed-size digest of content and canonical media type rather than recomputed from recency each request (#4532); appending a newer image therefore cannot demote and re-encode older images and bust Anthropic's prompt prefix cache. Position bytes count toward the shared retained-memory budget but are pinned — the shared evictor clears normalization cache slots first and never drops a position mid-request; positions shrink only through the store's own entry-count cap at request end. Unseen images still take the age-tier pyramid's first position, the total byte budget still binds, and a 413 `tierBias` retry still applies. Recorded positions only move down the ladder, so the store is monotonic. | | Adapter execution support | `src/adapters/run-turn-queue.ts`, `src/adapters/tool-catalog-nudge.ts`, `src/adapters/identity.ts`, `src/adapters/image.ts`, `src/adapters/upstream-http-error.ts` | Shared machinery: turn ordering, tool-catalog nudging, client fingerprinting, image conversion, upstream error normalization. | diff --git a/structure/transports/responses-failover.md b/structure/transports/responses-failover.md index 445b8a6c8a2..ee1fa25d578 100644 --- a/structure/transports/responses-failover.md +++ b/structure/transports/responses-failover.md @@ -553,7 +553,7 @@ stable ceiling without making a cooled account eligible. Same-provider auth reco physical target, rather than a diagnostic key, and a real send is charged once even when recovery rebuilds the request. -Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The fetch web-search sidecar captures pool activation before dispatch and applies the same Google-adapter, replayability, cancellation, failover-limit and pre-output gates to a complete 4 KiB-bounded raw envelope normalized through the existing Google error normalizer. Only structured `error.details[].reason === "VALIDATION_REQUIRED"` authorizes one sibling rotation with recovery kind `oauth-account-403`; sentence-only errors do not rotate. It carries the sibling's full token/project snapshot without persisting a needs-reauth mark. Its 403 and 429 replays hand the reserved permit to the adapter physical-send owner, charging each physical send once. The loop retains the bounded 403 until the sibling returns headers; construction or dispatch failure uses the original refusal's safe formatter, while cancellation and header deadlines retain lifecycle ownership. Unused permits are refunded before dispatch: iteration cleanup owns unclaimed permits, and the adapter physical-send owner releases claimed permits after failed admission. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). +Antigravity main Google adapter dispatch: after same-account refresh, a second pre-output 401 or terminal refresh failure may switch once to a live sibling within existing budgets. A pre-output 403 switches once only when the bounded Google adapter normalization found a complete structured `VALIDATION_REQUIRED` reason and `src/server/responses/antigravity-validation-refusal.ts` recognizes its exact marker. The 401 and 403 paths share one per-request sibling-attempt guard and carry the sibling's full token/project snapshot; an ineligible sibling, cancellation or budget refusal preserves the 403. The fetch web-search sidecar captures pool activation before dispatch and applies the same Google-adapter, replayability, cancellation, failover-limit and pre-output gates to a complete 4 KiB-bounded raw envelope normalized through the existing Google error normalizer. Only structured `error.details[].reason === "VALIDATION_REQUIRED"` authorizes one sibling rotation with recovery kind `oauth-account-403`; sentence-only errors do not rotate. It carries the sibling's full token/project snapshot without persisting a needs-reauth mark. Its 403 and 429 replays hand the reserved permit to the adapter physical-send owner, charging each physical send once. The loop retains the bounded 403 until the sibling returns headers; construction or dispatch failure uses the original refusal's safe formatter, while cancellation and header deadlines retain lifecycle ownership. Unused permits are refunded before dispatch: iteration cleanup owns unclaimed permits, and the adapter physical-send owner releases claimed permits after failed admission. Other sidecar paths, continuations and passthrough do not use this rotation (contract: `docs-site/src/content/docs/reference/configuration/providers.md`, `rotateAntigravityAccountOnAuthRefusal` in `src/oauth/generic-account-failover.ts`). Antigravity main Google adapter model-404 recovery in `src/server/responses/antigravity-model-denial-recovery.ts` accepts only an original, complete 4 KiB-bounded CCA 404 with structured NOT_FOUND evidence with a model ResourceInfo whose resourceName is exactly `models/`; message text alone is insufficient. `src/adapters/google-http.ts` classifies before display normalization and `src/adapters/antigravity-refusal.ts` carries process-local provenance through pacing wrappers; generic sentences, non-model resource errors, oversized, incomplete and erroring bodies do not deny or rotate. Non-abort body read failures preserve the original status with status-only text; cancellation propagates. The denial is scoped to captured config root, the sent credential generation and request observation order; a newer positive roster observation wins even within the same millisecond, and older same-generation roster publications are ignored before mutation. Its 24-hour TTL is independent of the six-hour discovery roster. Recovery tries unattempted eligible accounts within existing send/failover budgets, carrying token/project together through shared admission. Exhaustion names the model and distinguishes no eligible account from resolution, admission or send-budget failure. This model-specific denial does not mark the account for reauthentication or change Argon display quota. Precommit Codex model refusals use bounded account recovery for HTTP `detail` and WebSocket-projected `error.message` bodies. Only an exact HTTP 400 refusal naming the requested or wire model establishes diff --git a/tests/adapters/google/antigravity-routing-carry.test.ts b/tests/adapters/google/antigravity-routing-carry.test.ts new file mode 100644 index 00000000000..860d8a909a1 --- /dev/null +++ b/tests/adapters/google/antigravity-routing-carry.test.ts @@ -0,0 +1,255 @@ +import { afterEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gatherRoutedModels } from "../../../src/codex/catalog"; +import { clearModelCache } from "../../../src/codex/model-cache"; +import * as outbound from "../../../src/lib/provider-outbound"; +import { credentialGeneration, getAccountCredentialWithStatus } from "../../../src/oauth/store"; +import { preferredInitialAccount, routeAntigravityModel, clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; +import { + ANTIGRAVITY_MODEL_DENIAL_TTL_MS, antigravityAccountSupportsModel, antigravityAccountRosterModelIds, + captureAntigravityModelObservation, clearAntigravityAccountRosters, + recordAntigravityAccountRoster, recordAntigravityModelDenial, +} from "../../../src/providers/antigravity-account-roster"; +import { MODEL_DISCOVERY_MAX_MODELS } from "../../../src/providers/model-discovery"; +import { parseAntigravityAvailableModels } from "../../../src/providers/antigravity-models"; +import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; +import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { withStubbedProviderFetch } from "../../helpers/catalog-provider-fetch"; + +const provider = "google-antigravity"; +const originalHome = process.env.OPENCODEX_HOME; +const originalFetch = globalThis.fetch; +const homes: string[] = []; +const model = "claude-sonnet-4-6"; +const modelRows = (id: string) => [{ id, wireModelId: id }]; +const payload = (...ids: string[]) => ({ models: Object.fromEntries(ids.map(id => [id, {}])), + agentModelSorts: [{ groups: [{ modelIds: ids }] }] }); +function setup() { + const home = mkdtempSync(join(tmpdir(), "ocx-antigravity-carry-")); + homes.push(home); + process.env.OPENCODEX_HOME = home; + const store = { [provider]: { activeAccountId: "a", accounts: ["a", "b", "c"].map(id => ({ id, + credential: { access: `synthetic-${id}`, refresh: `synthetic-refresh-${id}`, expires: Date.now() + 3600000, projectId: `project-${id}` }, + })) } }; + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + return { home, store, cfg: withStubbedProviderFetch({ providers: { [provider]: { + adapter: "google", authMode: "oauth", baseUrl: "https://daily-cloudcode-pa.googleapis.com", + googleMode: "cloud-code-assist", liveModels: true, models: [], oauthAccountFailover: { enabled: true }, + } } } as unknown as OcxConfig) }; +} +afterEach(() => { + globalThis.fetch = originalFetch; + clearModelCache(); clearAntigravityAccountRosters(); clearGenericFailoverHealth(); clearAccountQuotaCache(provider); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = originalHome; + for (const home of homes.splice(0)) removeTreeWithRetry(home); +}); + +test("refreshing discovery unions usable siblings while retaining each account's own roster", async () => { + const { cfg, store, home } = setup(); + (store[provider].accounts[2] as unknown as { paused: boolean }).paused = true; + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + const sends: string[] = []; + globalThis.fetch = (async (_input, init) => { + const auth = new Headers(init?.headers).get("authorization")!; + sends.push(auth); + return Response.json(payload(auth.endsWith("-a") ? "model-active" : "model-sibling")); + }) as typeof fetch; + const rows = await gatherRoutedModels(cfg); + expect(rows.filter(row => row.provider === provider).map(row => row.id).sort()).toEqual(["model-active", "model-sibling"]); + expect(sends).toEqual(["Bearer synthetic-a", "Bearer synthetic-b"]); + expect(antigravityAccountRosterModelIds(provider, "a")).toEqual(["model-active"]); + expect(antigravityAccountRosterModelIds(provider, "b")).toEqual(["model-sibling"]); +}); + +test("a root change during active discovery cannot start sibling probes in the new home", async () => { + const { cfg } = setup(); + const sends: string[] = []; + globalThis.fetch = (async (_input, init) => { + sends.push(new Headers(init?.headers).get("authorization")!); + if (sends.length === 1) setup(); + return Response.json(payload("model-old-home")); + }) as typeof fetch; + const rows = await gatherRoutedModels(cfg); + expect(sends).toEqual(["Bearer synthetic-a"]); + expect(rows.some(row => row.id === "model-old-home")).toBe(false); + expect(antigravityAccountRosterModelIds(provider, "b")).toBeUndefined(); +}); + +test("active discovery held by a sibling cannot erase a newer inference denial", async () => { + const { cfg } = setup(); + let releaseSibling!: () => void; + let siblingSent!: () => void; + const atSibling = new Promise(resolve => { siblingSent = resolve; }); + const held = new Promise(resolve => { releaseSibling = resolve; }); + globalThis.fetch = (async (_input, init) => { + if (new Headers(init?.headers).get("authorization")!.endsWith("-b")) { + siblingSent(); await held; + } + return Response.json(payload(model)); + }) as typeof fetch; + const pending = gatherRoutedModels(cfg); + await atSibling; + try { + const sent = { ...captureAntigravityModelObservation(), + generation: credentialGeneration(getAccountCredentialWithStatus(provider, "a")!.credential) }; + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(true); + } finally { releaseSibling(); } + await pending; + expect(antigravityAccountRosterModelIds(provider, "a")).toEqual([model]); + expect(antigravityAccountSupportsModel(provider, "a", model)).toBe(false); + expect(routeAntigravityModel(cfg, provider, "a", model)).toEqual({ kind: "reroute", accountId: "b" }); + expect(preferredInitialAccount(cfg, provider, Date.now(), model)).toBe("b"); +}); + +test("active discovery held by a sibling cannot publish into replacement credentials", async () => { + const { cfg, store, home } = setup(); + globalThis.fetch = (async (_input, init) => { + const active = new Headers(init?.headers).get("authorization")!.endsWith("-a"); + if (!active) { + store[provider].accounts[0]!.credential.access = "synthetic-replacement"; + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + } + return Response.json(payload(active ? "model-active" : "model-sibling")); + }) as typeof fetch; + const rows = await gatherRoutedModels(cfg); + expect(rows.some(row => row.id === "model-active")).toBe(false); + expect(antigravityAccountRosterModelIds(provider, "a")).toBeUndefined(); +}); + +test("late D1 publication cannot roll back D2 before a delayed inference denial", async () => { + const { cfg } = setup(); + let releaseSibling!: () => void; + let siblingSent!: () => void; + const atSibling = new Promise(resolve => { siblingSent = resolve; }); + const held = new Promise(resolve => { releaseSibling = resolve; }); + globalThis.fetch = (async (_input, init) => { + if (new Headers(init?.headers).get("authorization")!.endsWith("-b")) { + siblingSent(); await held; + } + return Response.json(payload(model)); + }) as typeof fetch; + const d1 = gatherRoutedModels(cfg); + await atSibling; + const sent = { ...captureAntigravityModelObservation(), + generation: credentialGeneration(getAccountCredentialWithStatus(provider, "a")!.credential) }; + try { + recordAntigravityAccountRoster(provider, "a", modelRows(model)); // D2 completes first. + } finally { releaseSibling(); } + await d1; + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(false); + expect(antigravityAccountRosterModelIds(provider, "a")).toEqual([model]); + expect(antigravityAccountSupportsModel(provider, "a", model)).toBe(true); + expect(routeAntigravityModel(cfg, provider, "a", model)).toEqual({ kind: "keep" }); + expect(preferredInitialAccount(cfg, provider, Date.now(), model)).toBeNull(); // No switch from the eligible active account. +}); + +for (const timing of ["before-send", "before-publication"] as const) { + for (const mutation of ["paused", "removed", "reauth", "replaced", "root"] as const) { + test(`sibling ${mutation} ${timing} contributes no stale discovery`, async () => { + const { cfg, store, home } = setup(); + store[provider].accounts.splice(2); + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + const mutate = () => { + const b = store[provider].accounts[1]!; + if (mutation === "removed") store[provider].accounts.splice(1); + if (mutation === "paused") Object.assign(b, { paused: true }); + if (mutation === "reauth") Object.assign(b, { needsReauth: true }); + if (mutation === "replaced") b.credential.access = "synthetic-replacement"; + if (mutation === "root") { setup(); return; } + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + }; + const sends: string[] = []; + globalThis.fetch = (async (_input, init) => { + const auth = new Headers(init?.headers).get("authorization")!; + sends.push(auth); + if (auth.endsWith("-b") && timing === "before-publication") mutate(); + return Response.json(payload(auth.endsWith("-a") ? "model-active" : "model-sibling")); + }) as typeof fetch; + const realPost = outbound.providerOutboundPost; + const hook = spyOn(outbound, "providerOutboundPost").mockImplementation(async (...args) => { + if (new Headers(args[3].headers).get("authorization")?.endsWith("-b") && timing === "before-send") { + await Promise.resolve(); mutate(); + } + return realPost(...args); + }); + try { + const rows = await gatherRoutedModels(cfg); + expect(rows.some(row => row.id === "model-sibling")).toBe(false); + expect(antigravityAccountRosterModelIds(provider, "b")).toBeUndefined(); + if (timing === "before-send") expect(sends).toEqual(["Bearer synthetic-a"]); + } finally { hook.mockRestore(); } + }); + } +} + +test("disabled ids do not consume the enabled limit, including tier and image indexes", () => { + const body = payload("flagged", "legacy-disabled", "enabled"); + body.models.flagged = { disabled: true }; + expect(parseAntigravityAvailableModels(body, 1)?.map(row => row.id)).toEqual(["enabled"]); + expect(parseAntigravityAvailableModels(payload("enabled", "other"), 1)).toBeNull(); + expect(parseAntigravityAvailableModels({ ...payload("legacy-disabled"), + models: { "legacy-disabled": {}, "gemini-3.1-flash-image": {}, "future-tiered": { disabled: true } }, + imageGenerationModelIds: ["gemini-3.1-flash-image"], tieredModelIds: { flash: ["future-tiered"] }, + }, 1)?.map(row => row.id)).toEqual(["gemini-3.1-flash-image"]); +}); + +test("raw disabled responses remain independently bounded", () => { + const body = payload("legacy-disabled"); + body.agentModelSorts[0]!.groups[0]!.modelIds = Array(MODEL_DISCOVERY_MAX_MODELS * 4 + 1).fill("legacy-disabled"); + expect(parseAntigravityAvailableModels(body, 1)).toBeNull(); +}); + +test("denials last 24h independently of roster TTL and a newer positive observation wins", () => { + setup(); + expect(ANTIGRAVITY_MODEL_DENIAL_TTL_MS).toBe(24 * 60 * 60 * 1000); + const generation = credentialGeneration(getAccountCredentialWithStatus(provider, "a")!.credential); + const sent = { ...captureAntigravityModelObservation(), generation }; + expect(recordAntigravityModelDenial(provider, "a", model, sent, 1000)).toBe(true); + expect(antigravityAccountSupportsModel(provider, "a", model, 1000 + ANTIGRAVITY_MODEL_DENIAL_TTL_MS)).toBe(false); + expect(antigravityAccountSupportsModel(provider, "a", model, 1001 + ANTIGRAVITY_MODEL_DENIAL_TTL_MS)).toBeUndefined(); + recordAntigravityAccountRoster(provider, "a", modelRows(model)); + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(false); + expect(antigravityAccountSupportsModel(provider, "a", model)).toBe(true); +}); + +test("a delayed denial cannot enter a replacement credential or a different config home", () => { + const { home } = setup(); + const sent = { ...captureAntigravityModelObservation(), generation: credentialGeneration(getAccountCredentialWithStatus(provider, "a")!.credential) }; + const store = JSON.parse(readFileSync(join(home, "auth.json"), "utf8")); + store[provider].accounts[0].credential.access = "synthetic-replaced"; + writeFileSync(join(home, "auth.json"), JSON.stringify(store)); + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(false); + recordAntigravityAccountRoster(provider, "a", modelRows(model)); + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(false); + expect(antigravityAccountSupportsModel(provider, "a", model)).toBe(true); + setup(); + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(false); +}); + +test("a newer roster omitting the requested model does not suppress valid model-denial recovery", () => { + setup(); + const sent = { ...captureAntigravityModelObservation(), generation: credentialGeneration(getAccountCredentialWithStatus(provider, "a")!.credential) }; + recordAntigravityAccountRoster(provider, "a", modelRows("other-model")); + expect(recordAntigravityModelDenial(provider, "a", model, sent)).toBe(true); + expect(antigravityAccountSupportsModel(provider, "a", model)).toBe(false); +}); + +test("model routing and proactive preference rank by headroom within support/readiness tiers", () => { + const { cfg } = setup(); + recordAntigravityAccountRoster(provider, "a", modelRows("other")); + for (const id of ["b", "c"]) recordAntigravityAccountRoster(provider, id, modelRows(model)); + for (const [id, percent] of [["b", 90], ["c", 10]] as const) + setCachedProviderAccountQuotaForTests(provider, id, { customWindows: [{ label: "Cla", percent }], fetchedAt: Date.now() }); + expect(routeAntigravityModel(cfg, provider, "a", model)).toEqual({ kind: "reroute", accountId: "c" }); + expect(preferredInitialAccount(cfg, provider, Date.now(), model)).toBe("c"); + clearAntigravityAccountRosters(provider, "c"); + expect(routeAntigravityModel(cfg, provider, "a", model)).toEqual({ kind: "reroute", accountId: "b" }); + const fill = cfg.providers[provider] as OcxProviderConfig; + cfg.pool = { kernel: true } as never; + fill.oauthAccountFailover = { enabled: true, strategy: "fill-first" }; + recordAntigravityAccountRoster(provider, "c", modelRows(model)); + expect(routeAntigravityModel(cfg, provider, "a", model)).toEqual({ kind: "reroute", accountId: "b" }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d790e5fb463..fcd940683aa 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -750,6 +750,8 @@ "generic-session-affinity.test.ts": "oauth", "server-generic-oauth-affinity.test.ts": "server", "server-antigravity-model-roster-routing.test.ts": "server", + "antigravity-routing-carry.test.ts": "adapters/google", + "server-antigravity-model-denial-recovery.test.ts": "server", "generic-oauth-combo-account-cooldown.test.ts": "oauth", "generic-oauth-failover.test.ts": "oauth", "generic-oauth-refusal-composition.test.ts": "oauth", "github-copilot-auto.test.ts": "providers/github-copilot", "github-copilot-auto-runtime.test.ts": "providers/github-copilot", diff --git a/tests/helpers/responses-core-source.ts b/tests/helpers/responses-core-source.ts index 23ce1c863dc..09d18981437 100644 --- a/tests/helpers/responses-core-source.ts +++ b/tests/helpers/responses-core-source.ts @@ -83,6 +83,7 @@ export const RESPONSES_CORE_MODULES = [ "run-turn-execution.ts", "adapter-dispatch.ts", "antigravity-validation-refusal.ts", + "antigravity-model-denial-recovery.ts", "adapter-continuation.ts", "adapter-delivery.ts", "anthropic-thinking-recovery.ts", diff --git a/tests/server/server-antigravity-model-denial-recovery.test.ts b/tests/server/server-antigravity-model-denial-recovery.test.ts new file mode 100644 index 00000000000..b16397dc4fa --- /dev/null +++ b/tests/server/server-antigravity-model-denial-recovery.test.ts @@ -0,0 +1,102 @@ +import { expect, test } from "bun:test"; +import { isAntigravityModelNotFoundRefusal } from "../../src/server/responses/antigravity-validation-refusal"; +import { fetchAntigravityWithRetry } from "../../src/adapters/google-http"; + +const model = "gemini-3.8-flash"; +async function normalized(body: string | ReadableStream, status = 404): Promise { + return fetchAntigravityWithRetry({ url: "https://daily-cloudcode-pa.googleapis.com/v1internal:generateContent", + method: "POST", headers: {}, body: JSON.stringify({ model }) }, { + executor: (async () => new Response(body, { status })) as typeof fetch, + }); +} +const envelope = (message = "Requested entity was not found.", resourceName = `models/${model}`) => JSON.stringify({ + error: { code: 404, status: "NOT_FOUND", message, details: [{ "@type": "type.googleapis.com/google.rpc.ResourceInfo", resourceName }] }, +}); + +test("CCA model 404 classification requires original model-specific evidence", async () => { + const text = "Requested entity was not found."; + const accepted = await normalized(envelope()); + expect(await isAntigravityModelNotFoundRefusal(accepted)).toBe(true); + expect(await accepted.text()).toContain(text); + for (const status of [400, 401, 403, 404, 429, 500]) + expect(await isAntigravityModelNotFoundRefusal(new Response(text, { status }))).toBe(false); + for (const body of ["not found", "Requested entity is missing", "x".repeat(4097) + text, text + "x".repeat(4097)]) + expect(await isAntigravityModelNotFoundRefusal(await normalized(body))).toBe(false); + expect(await isAntigravityModelNotFoundRefusal(await normalized(envelope()), AbortSignal.abort())).toBe(false); +}); + +test("ambiguous and mismatched original resource errors cannot authorize model denial", async () => { + for (const body of [JSON.stringify({ error: { status: "NOT_FOUND", message: "Requested entity was not found." } }), + envelope("Requested entity was not found.", "projects/synthetic-project"), + envelope("Requested entity was not found.", "models/other-model"), + envelope("Requested entity was not found.", `projects/synthetic-project/models/${model}`), + envelope(`Model '${model}' was not found.`, "projects/synthetic-project")]) + expect(await isAntigravityModelNotFoundRefusal(await normalized(body))).toBe(false); +}); + +test("incidental model mentions without ResourceInfo cannot authorize model denial", async () => { + for (const message of [ + `Project projects/p was not found while resolving models/${model}`, + `Model 'other-model' was not found while resolving models/${model}`, + `models/other-model not found; requested models/${model}`, + `Model '${model}' was not found.`, + ]) { + const response = await normalized(JSON.stringify({ error: { code: 404, status: "NOT_FOUND", message } })); + expect(response.status).toBe(404); + expect(await isAntigravityModelNotFoundRefusal(response)).toBe(false); + } +}); + +test("a 5 KiB original JSON model 404 cannot become trusted after Google normalization", async () => { + const payload = JSON.parse(envelope()); + payload.error.filler = "x".repeat(5000); + const result = await normalized(JSON.stringify(payload)); + expect(await isAntigravityModelNotFoundRefusal(result)).toBe(false); + expect(await result.text()).toBe("Antigravity invalid request: HTTP 404"); +}); + +test("the original refusal ceiling counts UTF-8 bytes and admits exactly 4096", async () => { + const payload = JSON.parse(envelope()); + payload.error.filler = ""; + payload.error.filler = "x".repeat(4096 - Buffer.byteLength(JSON.stringify(payload), "utf8")); + expect(Buffer.byteLength(JSON.stringify(payload), "utf8")).toBe(4096); + expect(await isAntigravityModelNotFoundRefusal(await normalized(JSON.stringify(payload)))).toBe(true); + payload.error.filler += "é"; + expect(await isAntigravityModelNotFoundRefusal(await normalized(JSON.stringify(payload)))).toBe(false); +}); + +test("an incomplete original model 404 stream cannot authorize rotation", async () => { + const response = await normalized(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode(envelope())); }, + cancel() {}, + })); + expect(await isAntigravityModelNotFoundRefusal(response)).toBe(false); + expect(await response.text()).toBe("Antigravity invalid request: HTTP 404"); +}); + +test("an erroring original 404 body preserves status without denial provenance", async () => { + const response = await normalized(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode(envelope())); }, + pull(controller) { controller.error(new TypeError("synthetic body reset")); }, + })); + expect(response.status).toBe(404); + expect(await isAntigravityModelNotFoundRefusal(response)).toBe(false); + expect(await response.text()).toBe("Antigravity invalid request: HTTP 404"); +}); + +test("cancellation while reading an original 404 body propagates by identity", async () => { + const controller = new AbortController(); + const reason = new Error("synthetic cancellation"); + let reading!: () => void; + const atRead = new Promise(resolve => { reading = resolve; }); + const pending = fetchAntigravityWithRetry({ url: "https://daily-cloudcode-pa.googleapis.com/v1internal:generateContent", + method: "POST", headers: {}, body: JSON.stringify({ model }) }, { + abortSignal: controller.signal, + executor: (async () => new Response(new ReadableStream({ + pull() { reading(); }, + }, { highWaterMark: 0 }), { status: 404 })) as typeof fetch, + }); + await atRead; + controller.abort(reason); + await expect(pending).rejects.toBe(reason); +}); diff --git a/tests/server/server-antigravity-model-roster-routing.test.ts b/tests/server/server-antigravity-model-roster-routing.test.ts index e751f53170f..1dc070b6ee0 100644 --- a/tests/server/server-antigravity-model-roster-routing.test.ts +++ b/tests/server/server-antigravity-model-roster-routing.test.ts @@ -8,15 +8,17 @@ import { antigravitySelectedAccountUnlistedMessage, clearGenericFailoverHealth, } from "../../src/oauth/generic-account-failover"; -import { getAccountSet, saveCredential, setAccountPaused, setActiveAccount } from "../../src/oauth/store"; +import { getAccountSet, saveCredential, setAccountPaused, setActiveAccount, getAccountCredentialWithStatus, mergeAccountCredential } from "../../src/oauth/store"; import { clearPoolRotationState } from "../../src/oauth/pool-kernel"; -import { clearAntigravityAccountRosters, recordAntigravityAccountRoster } from "../../src/providers/antigravity-account-roster"; +import { antigravityAccountSupportsModel, clearAntigravityAccountRosters, recordAntigravityAccountRoster } from "../../src/providers/antigravity-account-roster"; import { handleResponses } from "../../src/server/responses"; import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import * as retry from "../../src/lib/upstream-retry"; +import * as failover from "../../src/oauth/generic-account-failover"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; // #7500: an Antigravity request reaches only accounts whose own model discovery may serve it. const provider = "google-antigravity"; @@ -30,7 +32,7 @@ let originalFetch: typeof fetch; let sleep: ReturnType; let ids: string[]; let sends: string[]; -let onSend: ((auth: string) => Promise) | undefined; +let onSend: ((auth: string, wireModel: string) => Promise) | undefined; beforeEach(async () => { previousHome = process.env.OPENCODEX_HOME; @@ -58,7 +60,7 @@ beforeEach(async () => { throw new Error("Unexpected external request in Antigravity model routing regression"); const auth = new Headers(init?.headers).get("authorization")!; sends.push(auth); - return await onSend?.(auth) ?? Response.json({ response: { + return await onSend?.(auth, JSON.parse(String(init?.body)).model) ?? Response.json({ response: { candidates: [{ content: { role: "model", parts: [{ text: "synthetic result" }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 3, totalTokenCount: 8 }, } }); @@ -96,11 +98,11 @@ function rosters(...lists: Array): void { }); } -async function turn(cfg: OcxConfig): Promise<{ status: number; body: { error?: { message?: string; type?: string; code?: string } } }> { +async function turn(cfg: OcxConfig, options: Parameters[3] = {}): Promise<{ status: number; body: { error?: { message?: string; type?: string; code?: string } } }> { const response = await handleResponses(new Request("http://127.0.0.1/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: `${provider}/${model}`, input: "synthetic input", stream: false }), - }), cfg, { model, provider }, {}); + }), cfg, { model, provider }, options); return { status: response.status, body: await response.json() as never }; } @@ -155,3 +157,124 @@ test("a recovery rebind onto a newly selected omitting account is abandoned, not expect(result.status).not.toBe(200); expect(sends).not.toContain(bearer(1)); }); + +const notFound = (wireModel: string) => Response.json({ error: { code: 404, status: "NOT_FOUND", message: "Requested entity was not found.", + details: [{ "@type": "type.googleapis.com/google.rpc.ResourceInfo", resourceType: "aiplatform.googleapis.com/Model", resourceName: `models/${wireModel}` }], +} }, { status: 404 }); + +test("a bounded CCA model 404 rotates within the request and caches only that account's denial", async () => { + onSend = async (auth, wireModel) => auth === bearer(0) ? notFound(wireModel) : undefined; + expect((await turn(config(true))).status).toBe(200); + expect(sends).toEqual([bearer(0), bearer(1)]); + expect(antigravityAccountSupportsModel(provider, ids[0]!, model)).toBe(false); + expect(antigravityAccountSupportsModel(provider, ids[0]!, "unrelated-model")).toBeUndefined(); +}); + +test("all accounts denying a model gives a model-specific exhaustion without cycling", async () => { + onSend = async (_auth, wireModel) => notFound(wireModel); + const result = await turn(config(true)); + expect(result.status).toBe(404); + expect(sends).toEqual([bearer(0), bearer(1), bearer(2)]); + expect(result.body.error?.message).toContain(`No eligible stored Google Antigravity account can serve the model "${model}"`); +}); + +test("an unrelated 404 never rotates or records a model denial", async () => { + onSend = async () => Response.json({ error: { message: "route not found" } }, { status: 404 }); + expect((await turn(config(true))).status).toBe(404); + expect(sends).toEqual([bearer(0)]); + expect(antigravityAccountSupportsModel(provider, ids[0]!, model)).toBeUndefined(); +}); + +for (const subject of ["Project projects/p", "Model 'other-model'"]) { + test(`incidental model mention in a missing ${subject} never denies or rotates`, async () => { + onSend = async (_auth, wireModel) => Response.json({ error: { code: 404, status: "NOT_FOUND", + message: `${subject} was not found while resolving models/${wireModel}`, + } }, { status: 404 }); + expect((await turn(config(true))).status).toBe(404); + expect(sends).toEqual([bearer(0)]); + for (const id of ids) expect(antigravityAccountSupportsModel(provider, id, model)).toBeUndefined(); + onSend = undefined; + expect((await turn(config(true))).status).toBe(200); + expect(sends).toEqual([bearer(0), bearer(0)]); + }); +} + +test("an erroring 404 body stays 404 without denial, rotation or retry", async () => { + onSend = async () => new Response(new ReadableStream({ + start(controller) { controller.error(new TypeError("synthetic body reset")); }, + }), { status: 404 }); + const result = await turn(config(true)); + expect(result.status).toBe(404); + expect(result.body.error?.message).toContain("HTTP 404"); + expect(sends).toEqual([bearer(0)]); + for (const id of ids) expect(antigravityAccountSupportsModel(provider, id, model)).toBeUndefined(); +}); + +test("a generic resource 404 with the CCA sentence never poisons model eligibility or rotates", async () => { + onSend = async () => Response.json({ error: { code: 404, status: "NOT_FOUND", message: "Requested entity was not found.", + details: [{ "@type": "type.googleapis.com/google.rpc.ResourceInfo", resourceType: "cloudresourcemanager.googleapis.com/Project", + resourceName: "projects/synthetic-project-one" }], + } }, { status: 404 }); + const result = await turn(config(true)); + expect(result.status).toBe(404); + expect(result.body.error?.message).toContain("Requested entity was not found."); + expect(result.body.error?.message).not.toContain("No eligible stored"); + expect(sends).toEqual([bearer(0)]); + for (const id of ids) expect(antigravityAccountSupportsModel(provider, id, model)).toBeUndefined(); + onSend = undefined; + expect((await turn(config(true))).status).toBe(200); + expect(sends).toEqual([bearer(0), bearer(0)]); +}); + +test("a spent send budget preserves the 404 and reports recovery failure rather than no eligible account", async () => { + const cfg = config(true); + const sendBudget = createRequestExecutionBudget({ maxTotalModelSends: 1, baseSendAllowance: 1, + finalRecoveryAllowance: 0, maxAlternateTargetSends: 0, maxTargetTransitions: 0 }); + onSend = async (_auth, wireModel) => notFound(wireModel); + const result = await turn(cfg, { sendBudget }); + expect(result.status).toBe(404); + expect(sends).toEqual([bearer(0)]); + expect(result.body.error?.message).toContain("could not be resolved, admitted, or sent within the request budget"); + expect(result.body.error?.message).not.toContain("No eligible stored"); +}); + +for (const replacement of [false, true]) { + test(`a delayed old 404 cannot override newer positive discovery${replacement ? " for replacement credentials" : " in the same generation"}`, async () => { + let release404!: () => void; + let sent!: () => void; + const atSend = new Promise(resolve => { sent = resolve; }); + const delayed = new Promise(resolve => { release404 = resolve; }); + onSend = async (auth, wireModel) => { + if (auth !== bearer(0)) return undefined; + sent(); await delayed; return notFound(wireModel); + }; + const pending = turn(config(true)); + await atSend; + try { + if (replacement) { + const row = getAccountCredentialWithStatus(provider, ids[0]!)!; + await mergeAccountCredential(provider, ids[0]!, { ...row.credential, access: "synthetic-replacement", refresh: "synthetic-new-refresh" }); + } + rosters(true, undefined, undefined); + } finally { release404(); } + expect((await pending).status).toBe(404); + expect(sends).toEqual([bearer(0)]); + expect(antigravityAccountSupportsModel(provider, ids[0]!, model)).toBe(true); + }); +} + +test("replacement admission failure is reported separately from no eligible account", async () => { + let failure: ReturnType | undefined; + onSend = async (auth, wireModel) => { + if (auth !== bearer(0)) return undefined; + failure = spyOn(failover, "failoverAccountSnapshot").mockRejectedValue(new Error("synthetic resolution failure")); + return notFound(wireModel); + }; + try { + const result = await turn(config(true)); + expect(result.status).toBe(404); + expect(sends).toEqual([bearer(0)]); + expect(result.body.error?.message).toContain("could not be resolved, admitted, or sent within the request budget"); + expect(result.body.error?.message).not.toContain("No eligible stored"); + } finally { failure?.mockRestore(); } +});