diff --git a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md index a4eb1ec9722..0a7112cab84 100644 --- a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md +++ b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md @@ -48,6 +48,27 @@ that the account pool routed to the main account, the earliest known time anothe pool account recovers, whichever comes first (never sooner than one minute). Resume the Goal after that time; routing then picks an account from fresh state. +## Stored-account credit metadata retries + +An explicitly selected stored account can reach its included usage limit while its last +credit balance observation is older than five minutes. If credit spending is enabled for +that account and the evidence is stale, missing or incomplete, its local 429 advises retrying +after the 300-second base metadata cadence (or at the included reset if sooner). A later known upstream +cooldown for the selected quota scope, including account-wide holds, still takes precedence; when +both apply, the later deadline wins. Unrelated +model-group holds are not imported. The original included-window reset remains attached to the +internal refusal as `resetAt`; the shorter hint is a metadata recheck opportunity, not a new quota reset. + +The request remains refused until fresh spendable credit evidence is available. This hint +does not perform a refresh, bypass metadata-query pacing, guarantee recovery, change account +selection, or enable credit spending. Direct account bindings retain their identity. Explicit +credit opt-out, an older cached null without provenance, a known spending veto or a finite empty +balance without unlimited entitlement retain the included-reset advice. A newly observed missing +or malformed credit summary retracts spending authority but permits the short metadata retry; +an explicit spending veto still wins. Unlimited credits with a zero balance require fresh evidence +just like other credits. Ordinary upstream 429 delays keep the existing parser ceiling of 24 hours +for numeric and HTTP-date Retry-After values. + ## Triage: `idle timeout waiting for SSE` Codex ends a stream that has sent no event for its `stream_idle_timeout_ms` diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 3dac671e9ac..53debea0645 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -94,7 +94,7 @@ "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", - "codex-pool-credit-policy.test.ts": "codex-integration", + "codex-pool-credit-policy.test.ts": "codex-integration", "codex-credit-retry-after.test.ts": "codex-integration", "codex-credit-scoped-retry.test.ts": "codex-integration", "codex-sidecar-credit-policy.test.ts": "codex-integration", "codex-credits-after-limit-main.test.ts": "codex-integration", "service-runtime-preflight.test.ts": "service", "service-desktop-runtime-preflight.test.ts": "service", diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 5afdd2230fd..d6a52b96ae3 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1,4 +1,4 @@ -import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt } from "./pool-credit-policy"; +import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt, poolCreditRecoveryNeeded, poolContextCreditRecoveryNeeded, poolCreditRecoveryRetryAt } from "./pool-credit-policy"; import { hasSpendableCodexCredits } from "./quota-types"; import { nextQuotaQueryDelay } from "./quota-query-backoff"; import { noteMainAccountActivity } from "./main-account-external-usage"; @@ -548,17 +548,20 @@ export class CodexMainAccountHardLockError extends CodexAccountCooldownError { export class CodexPoolAccountCreditsOffError extends CodexAccountCooldownError { readonly resetAt: number; - constructor(accountId: string, resetAt: number) { - super(accountId, resetAt); + constructor(accountId: string, resetAt: number, creditRecovery = false, quotaScope?: CodexQuotaScope) { + // Keep the included reset; a metadata hint neither admits nor clears an upstream hold. + super(accountId, creditRecovery ? poolCreditRecoveryRetryAt(accountId, resetAt, quotaScope) : resetAt); this.name = "CodexPoolAccountCreditsOffError"; this.resetAt = resetAt; - this.message = "The selected Codex account reached its usage limit, and spending credits is off or no fresh spendable balance is available." + this.message = creditRecovery + ? "The selected Codex account reached its usage limit and needs a fresh credit observation. Retry later; spending remains blocked until fresh spendable credits are observed." + : "The selected Codex account reached its usage limit, and spending credits is off or no fresh spendable balance is available." + " Wait for the limit to reset, choose another account, or explicitly allow credits for this account in Codex Auth."; } } -function assertPoolAccountCredits(accountId: string, resetAt: number | undefined): void { - if (resetAt !== undefined) throw new CodexPoolAccountCreditsOffError(accountId, resetAt); +function assertPoolAccountCredits(accountId: string, resetAt: number | undefined, creditRecovery: boolean, quotaScope?: CodexQuotaScope): void { + if (resetAt !== undefined) throw new CodexPoolAccountCreditsOffError(accountId, resetAt, creditRecovery, quotaScope); } /** The main login may not spend credits and one of its usage windows is full (#6334). */ @@ -780,7 +783,7 @@ export function createCodexAuthDispatchGuard( return headers => { if (config.pausedCodexAccountIds?.includes(ctx.accountId)) throw new CodexPoolAuthenticationError("Selected Codex account is paused; resume it in account settings"); if (ctx.kind === "main-pool") { reserveGuard?.(headers); return; } - assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, config)); + assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, config), poolContextCreditRecoveryNeeded(ctx, config), ctx.quotaScope); reserveGuard?.(headers); }; } @@ -1569,7 +1572,7 @@ async function resolveCodexAuthContextUnannotated( try { assertCodexAccountValidationReady(accountId); if (configuredCodexMainDuplicate(config, accountId)) throw new CodexPoolAuthenticationError(CODEX_MAIN_DUPLICATE_ACTION); - assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config)); + assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config), poolCreditRecoveryNeeded(policy, accountId), quotaScope); } catch (cause) { // Nothing will reach upstream, so give the trial back instead of leaving the held account // unprobeable until the lease deadline lapses (#4701). @@ -1700,7 +1703,7 @@ async function resolveCodexAuthContextUnannotated( const token = await getValidCodexToken(accountId, { signal: options.signal }); assertCodexAccountValidationReady(accountId); if (configuredCodexMainDuplicate(config, accountId)) throw new CodexPoolAuthenticationError(CODEX_MAIN_DUPLICATE_ACTION); - assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config)); + assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config), poolCreditRecoveryNeeded(policy, accountId), quotaScope); return bindPoolCreditPolicy({ kind: "pool", accountId, @@ -1732,7 +1735,7 @@ async function resolveCodexAuthContextUnannotated( export function assertCodexAuthContextNotCooled(ctx: CodexAuthContext | undefined): void { if (ctx?.kind !== "pool" && ctx?.kind !== "main-pool") return; - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx), poolContextCreditRecoveryNeeded(ctx), ctx.quotaScope); // A context holding the probe lease was deliberately admitted through the cooldown. if (ctx.probeLeaseId) return; const cooldown = getCodexQuotaHealthSnapshot(ctx.accountId, ctx.quotaScope); @@ -1748,7 +1751,7 @@ export function applyCodexAuthContextToProvider( ): OcxRuntimeProviderConfig { if (mode !== "pool" || (ctx.kind !== "pool" && ctx.kind !== "main-pool") || provider.authMode !== "forward") return provider; assertCodexAccountValidationReady(ctx.accountId); - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx), poolContextCreditRecoveryNeeded(ctx), ctx.quotaScope); return { ...provider, _codexAccountOverride: { @@ -1803,7 +1806,7 @@ export function materializeCodexUpstreamAuth( } if (ctx.kind === "pool" || ctx.kind === "main-pool") { assertCodexAccountValidationReady(ctx.accountId); - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, options.config)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, options.config), poolContextCreditRecoveryNeeded(ctx, options.config), ctx.quotaScope); selected.set("authorization", `Bearer ${ctx.accessToken}`); selected.set("chatgpt-account-id", ctx.chatgptAccountId); if (ctx.kind === "main-pool") { diff --git a/src/codex/pool-credit-policy.ts b/src/codex/pool-credit-policy.ts index d1a69e1eadd..e67898626f3 100644 --- a/src/codex/pool-credit-policy.ts +++ b/src/codex/pool-credit-policy.ts @@ -1,8 +1,11 @@ import type { OcxConfig } from "../types"; -import { codexUsageLimitResetAt, isCodexAccountHeldForCredits } from "./account-credit-use"; +import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt, isCodexAccountHeldForCredits } from "./account-credit-use"; import { MAIN_CODEX_ACCOUNT_ID } from "./main-account"; import { isSelectableCodexPoolAccount } from "./account-id"; import { getAccountQuota } from "./quota"; +import { hasSpendableCodexCredits } from "./quota-types"; +import { getCodexAccountCooldownUntil, scopedHealthFor, type CodexQuotaScope } from "./routing/health-store"; +import { nextQuotaQueryDelay } from "./quota-query-backoff"; export type PoolCreditPolicyConfig = Readonly>; type PoolContext = { kind: "pool"; accountId: string }; @@ -65,3 +68,34 @@ export function poolContextCreditHoldResetAt( const catalog = override?.codexAccounts ? override : bound?.config ?? policy; return poolCreditHoldResetAt(policy, context.accountId, catalog, now); } + +/** Retry advice only: absent/uncertain evidence can be re-observed, never spent. */ +export function poolCreditRecoveryNeeded(policy: PoolCreditPolicyConfig, accountId: string, now = Date.now()): boolean { + if (!codexAccountUsesCreditsAfterLimit(policy, accountId)) return false; + const quota = getAccountQuota(accountId); + const credits = quota?.credits; + // Only parser-proven unavailable null is re-observable. Legacy null and known + // vetoes remain conservative; unlimited overrides balance, never an explicit veto. + if (credits === null) return quota?.creditObservationUnavailable === true; + if (credits?.allowed === false || credits?.overageLimitReached === true + || credits?.hasCredits === false || (credits?.unlimited !== true && typeof credits?.balance === "number" + && Number.isFinite(credits.balance) && credits.balance <= 0)) return false; + return !hasSpendableCodexCredits({ credits }, now); +} + +/** Use the same live policy/override as the context's admission check. */ +export function poolContextCreditRecoveryNeeded(context: PoolContext, override?: PoolCreditPolicyConfig): boolean { + const policy = override ?? policies.get(context)?.policy; + return !!policy && poolCreditRecoveryNeeded(policy, context.accountId); +} + +/** Retry advice observes both applicable holds without changing routing's health precedence. */ +export function poolCreditRecoveryRetryAt( + accountId: string, resetAt: number, quotaScope?: CodexQuotaScope, now = Date.now(), +): number { + const scopedUntil = quotaScope ? scopedHealthFor(accountId, quotaScope)?.cooldownUntil : undefined; + const liveScopedUntil = typeof scopedUntil === "number" && Number.isFinite(scopedUntil) && scopedUntil > now + ? scopedUntil : 0; + return Math.max(Math.min(resetAt, now + nextQuotaQueryDelay()), + getCodexAccountCooldownUntil(accountId, now) ?? 0, liveScopedUntil); +} diff --git a/src/codex/quota-types.ts b/src/codex/quota-types.ts index d3854172f8f..be1a0bb65c4 100644 --- a/src/codex/quota-types.ts +++ b/src/codex/quota-types.ts @@ -108,6 +108,9 @@ export type StoredAccountQuota = { resetCredits?: number; /** Spendable usage credits; resetCredits are separate, manually redeemed reset tickets. */ credits?: CodexSpendableCredits | null; + /** Only with null: provider evidence was unavailable, not a proven spending veto. + * Affects retry advice only; legacy null remains conservative. */ + creditObservationUnavailable?: true; /** Monthly usage came from an explicitly monthly PRIMARY, not supplementary tertiary, window. */ monthlyIsPrimaryWindow?: boolean; updatedAt: number; diff --git a/src/codex/quota.ts b/src/codex/quota.ts index ebcd30372aa..5eebba7782b 100644 --- a/src/codex/quota.ts +++ b/src/codex/quota.ts @@ -389,8 +389,11 @@ function mergeAccountQuota( const creditsOnly = snapshotHasCredits(quota) && !snapshotHasUsage(quota); // An explicit empty/invalid credits summary retracts the old balance. Partial headers // retain the original observation clock instead of extending its spending authority. - if (quota.credits !== undefined) next.credits = quota.credits; - else if (existing?.credits !== undefined) next.credits = existing.credits; + const creditSource = quota.credits !== undefined ? quota : existing; + if (creditSource?.credits !== undefined) next.credits = creditSource.credits; + if (creditSource?.credits === null && creditSource.creditObservationUnavailable === true) { + next.creditObservationUnavailable = true; + } if (creditsOnly) { if (existing?.weeklyPercent !== undefined) next.weeklyPercent = existing.weeklyPercent; @@ -680,6 +683,8 @@ export function updateAccountQuota( ...(existing?.customWindows !== undefined ? { customWindows: existing.customWindows } : {}), ...(existing?.resetCredits !== undefined ? { resetCredits: existing.resetCredits } : {}), ...(existing?.credits !== undefined ? { credits: existing.credits } : {}), + ...(existing?.credits === null && existing.creditObservationUnavailable === true + ? { creditObservationUnavailable: true as const } : {}), updatedAt: Date.now(), }; assignCarriedShort(quota, existing, quota.updatedAt); @@ -739,7 +744,10 @@ function readMainPolicyQuota(value: unknown): MainPolicyQuota | null { } if (quota.monthlyPercent !== undefined && raw.monthlyIsPrimaryWindow === true) quota.monthlyIsPrimaryWindow = true; // Retain the independent credit timestamp on restart; it still expires at its original deadline. - if (raw.credits === null) quota.credits = null; + if (raw.credits === null) { + quota.credits = null; + if (raw.creditObservationUnavailable === true) quota.creditObservationUnavailable = true; + } else if (raw.credits && typeof raw.credits === "object" && !Array.isArray(raw.credits)) { const stored = raw.credits as Record; if (typeof stored.observedAt === "number" && Number.isFinite(stored.observedAt) && stored.observedAt >= 0) { @@ -976,6 +984,9 @@ export function parseUsageQuota(data: WhamUsageResponse): Omit; +let originalFetch: typeof fetch; +let networkCalls = 0; +const cfg = (): OcxConfig => ({ + port: 10100, defaultProvider: "openai", providers: {}, codexMainAccountHardLock: false, + codexAccounts: [{ id: ID, email: "fixture@example.test", plan: "plus", isMain: false }], + codexAccountNamespaces: { subAlias: ID }, creditCodexAccountIds: [ID], + activeCodexAccountId: ID, autoSwitchThreshold: 0, upstreamFailoverThreshold: 3, +}); +function observe(credits: NonNullable[1]>["credits"], resetAt = RESET) { + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: resetAt, ...(credits === undefined ? {} : { credits }) }); +} +const positive = (age = 360_000) => ({ hasCredits: true, balance: 5, allowed: true, observedAt: NOW - age }); +const resolve = (config = cfg()) => resolveCodexAuthContext(new Headers(), config, "direct", { accountId: ID, modelId: "gpt-5.5" }); +async function refusal(config = cfg()): Promise { + const error = await resolve(config).catch(e => e); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(RESET); + expect(isAccountNeedsReauth(ID)).toBe(false); + return error; +} +function expectRetry(error: CodexAccountCooldownError, seconds: number) { + const response = cooldownErrorResponse(error, Date.now()); + expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe(String(seconds)); +} +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; previousCodexHome = process.env.CODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-credit-retry-")); + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + clock = spyOn(Date, "now").mockReturnValue(NOW); + const ok = { success: true, exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + setIcaclsRunnerForTests(() => ok); setAsyncIcaclsRunnerForTests(async () => ok); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + saveCodexAccountCredential(ID, { accessToken: "fixture-credit-access", refreshToken: "fixture-credit-refresh", + expiresAt: NOW + 3_600_000, chatgptAccountId: "fixture-credit-workspace" }); + originalFetch = globalThis.fetch; networkCalls = 0; + globalThis.fetch = Object.assign((): never => { networkCalls++; throw new Error("Unexpected network request"); }, + { preconnect: originalFetch.preconnect }); +}); +afterEach(async () => { + try { + expect(networkCalls).toBe(0); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + await flushConfigDirHardeningForTests(); + } finally { + globalThis.fetch = originalFetch; clock.mockRestore(); + setIcaclsRunnerForTests(null); setAsyncIcaclsRunnerForTests(null); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); + } +}); +for (const kind of ["stale", "missing", "partial", "future", "invalid-time"] as const) test(`opted-in exact Direct account gets a metadata retry for ${kind} evidence`, async () => { + observe(kind === "stale" ? positive() : kind === "future" ? positive(-1) + : kind === "invalid-time" ? { ...positive(), observedAt: NaN } + : kind === "partial" ? { observedAt: NOW } : undefined); + const before = structuredClone(getAccountQuota(ID)); + const error = await refusal(); expectRetry(error, 300); + expect(getAccountQuota(ID)).toEqual(before); + await expect(resolve()).rejects.toBeInstanceOf(CodexPoolAccountCreditsOffError); +}); +test("opt-out retains the actual six-day reset despite stale evidence", async () => { + observe(positive()); const config = cfg(); config.creditCodexAccountIds = []; + expectRetry(await refusal(config), 518_400); +}); +for (const age of [0, 360_000]) for (const [reason, veto] of [ + ["empty", { balance: 0 }], ["spend control", { allowed: false }], + ["overage", { overageLimitReached: true }], ["no credits", { hasCredits: false }], +] as const) test(`known ${reason} retains reset at credit age ${age}`, async () => { + observe({ ...positive(age), ...veto }); expectRetry(await refusal(), 518_400); +}); +test("explicit retraction retains the included reset", async () => { + observe(null); expectRetry(await refusal(), 518_400); +}); +test("spend-control-only revocation remains a reset-bound refusal", async () => { + observe(positive(0)); setAccountQuotaFromParsed(ID, parseUsageQuota({ spend_control: { reached: true } })); + expect(getAccountQuota(ID)?.credits).toBeNull(); expectRetry(await refusal(), 518_400); +}); +test("the included reset earlier than metadata cadence remains the hint", async () => { + observe(positive(), NOW + 30_000); + const error = await resolve().catch(e => e); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(NOW + 30_000); expectRetry(error, 30); +}); +test("TTL boundary admits at five minutes and refuses just beyond it", async () => { + observe(positive(300_000)); const context = await resolve(); expect(context.kind).toBe("pool"); + clock.mockReturnValue(NOW + 1); + expect(() => materializeCodexUpstreamAuth(new Headers(), context)).toThrow(CodexPoolAccountCreditsOffError); +}); +for (const boundary of ["materialize", "dispatch", "sidecar", "provider", "cooldown"] as const) test(`late ${boundary} refusal retains live consent and metadata hint`, async () => { + observe(positive(0)); const config = cfg(); const context = await resolve(config); + observe(positive()); + const run = () => boundary === "materialize" ? materializeCodexUpstreamAuth(new Headers(), context) + : boundary === "dispatch" ? createCodexAuthDispatchGuard(context, config, "gpt-5.5")!(new Headers()) + : boundary === "provider" ? applyCodexAuthContextToProvider({ adapter: "openai-responses", baseUrl: "https://example.test", authMode: "forward" }, context, "pool") + : boundary === "cooldown" ? assertCodexAuthContextNotCooled(context) + : createOpenAiSidecarCreditGuard(context, config)!(); + let error: unknown; try { run(); } catch (e) { error = e; } + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); expectRetry(error as CodexAccountCooldownError, 300); + config.creditCodexAccountIds = []; + try { run(); } catch (e) { error = e; } + expectRetry(error as CodexAccountCooldownError, 518_400); +}); +test("authoritative upstream 429 Retry-After is not capped", () => { + const error = new CodexAccountCooldownError(ID, NOW + 3_600_000, "retry-after"); + expectRetry(error, 3_600); +}); +test("a stored upstream 429 cooldown still dominates a metadata-only retry hint", async () => { + observe(positive()); + recordCodexUpstreamOutcome(cfg(), ID, 429, { retryAfter: "3600", now: NOW }); + expectRetry(await refusal(), 3_600); +}); + +test("materialization override wins over bound credit consent in both directions", async () => { + observe(positive(0)); const config = cfg(); const context = await resolve(config); observe(positive()); + const attempt = (allowed: boolean) => { + try { materializeCodexUpstreamAuth(new Headers(), context, { config: { creditCodexAccountIds: allowed ? [ID] : [] } }); } + catch (error) { expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); return error as CodexPoolAccountCreditsOffError; } + throw new Error("expected credit admission refusal"); + }; + expectRetry(attempt(false), 518_400); + config.creditCodexAccountIds = []; + const error = attempt(true); expect(error.resetAt).toBe(RESET); expectRetry(error, 300); +}); + +test("unlimited zero balance admits while fresh and requests metadata after expiry", async () => { + observe({ ...positive(300_000), unlimited: true, balance: 0 }); + expect((await resolve()).kind).toBe("pool"); + clock.mockReturnValue(NOW + 1); + expectRetry(await refusal(), 300); +}); +for (const veto of [{ allowed: false }, { hasCredits: false }, { overageLimitReached: true }]) { + test(`unlimited zero balance preserves explicit veto ${JSON.stringify(veto)}`, async () => { + observe({ ...positive(), unlimited: true, balance: 0, ...veto }); + expectRetry(await refusal(), 518_400); + }); +} +const unavailablePayloads: Parameters[0][] = [ + { credits: {} }, { credits: null }, { credits: { balance: "invalid" } }, + { rate_limit: { allowed: false } }, + { credits: "invalid" } as unknown as Parameters[0], +]; +for (const payload of unavailablePayloads) test(`parser uncertainty stays refused with a short retry: ${JSON.stringify(payload)}`, async () => { + observe(positive(0)); + setAccountQuotaFromParsed(ID, parseUsageQuota(payload)); + expect(getAccountQuota(ID)?.credits).toBeNull(); + expectRetry(await refusal(), 300); + updateAccountQuota(ID, 100, RESET); + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: RESET }); + expectRetry(await refusal(), 300); + const config = cfg(); config.creditCodexAccountIds = []; + expectRetry(await refusal(config), 518_400); +}); +for (const payload of unavailablePayloads) for (const spend_control of [{ reached: true }, {}]) { + test(`spending veto dominates unavailable evidence: ${JSON.stringify({ ...payload, spend_control })}`, async () => { + observe(positive(0)); + setAccountQuotaFromParsed(ID, parseUsageQuota({ ...payload, spend_control })); + expectRetry(await refusal(), 518_400); + }); +} +test("replacement clears uncertainty provenance, without reviving credit authority", async () => { + observe(positive(0)); + const context = await resolve(); + const unknown = () => setAccountQuotaFromParsed(ID, parseUsageQuota({ credits: {} })); + unknown(); expectRetry(await refusal(), 300); + expect(() => materializeCodexUpstreamAuth(new Headers(), context)).toThrow(CodexPoolAccountCreditsOffError); + observe(null); expectRetry(await refusal(), 518_400); + unknown(); observe({ ...positive(), allowed: false }); expectRetry(await refusal(), 518_400); + unknown(); observe(positive(0)); expect((await resolve()).kind).toBe("pool"); + expect(getAccountQuota(ID)?.creditObservationUnavailable).toBeUndefined(); + unknown(); observe(positive()); expectRetry(await refusal(), 300); + expect(getAccountQuota(ID)?.creditObservationUnavailable).toBeUndefined(); + unknown(); setAccountQuotaFromParsed(ID, parseUsageQuota({ spend_control: { reached: true } })); + expectRetry(await refusal(), 518_400); + unknown(); expectRetry(await refusal(), 300); +}); +test("unavailable versus legacy null survives persistence and a fresh reader process", async () => { + observe(positive()); + setAccountQuotaFromParsed(ID, parseUsageQuota({ credits: {} })); + setAccountQuotaFromParsed("legacy-null", { credits: null }); + const path = join(home, "codex-quota-cache.json"); + for (let tries = 0; !existsSync(path) && tries < 100; tries++) await Bun.sleep(10); + expect(existsSync(path)).toBe(true); + const child = Bun.spawnSync([process.execPath, "-e", ` + import { getAccountQuota } from './src/codex/quota'; + import { poolCreditRecoveryNeeded } from './src/codex/pool-credit-policy'; + Date.now = () => ${NOW}; + const policy = {creditCodexAccountIds: ['${ID}', 'legacy-null']}; + console.log(JSON.stringify(['${ID}', 'legacy-null'].map(id => ({ + credits: getAccountQuota(id)?.credits, recovery: poolCreditRecoveryNeeded(policy, id) + })))); + `], { cwd: repoRoot(), env: process.env, stdout: "pipe", stderr: "pipe" }); + expect(child.exitCode).toBe(0); + expect(JSON.parse(child.stdout.toString())).toEqual([ + { credits: null, recovery: true }, { credits: null, recovery: false }, + ]); +}); diff --git a/tests/codex-integration/codex-credit-scoped-retry.test.ts b/tests/codex-integration/codex-credit-scoped-retry.test.ts new file mode 100644 index 00000000000..7c44f9a2713 --- /dev/null +++ b/tests/codex-integration/codex-credit-scoped-retry.test.ts @@ -0,0 +1,150 @@ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OcxConfig } from "../../src/types"; +import { + CodexPoolAccountCreditsOffError, cooldownErrorResponse, resolveCodexAuthContext, + materializeCodexUpstreamAuth, materializeCodexUpstreamAuthAsync, + createCodexAuthDispatchGuard, applyCodexAuthContextToProvider, assertCodexAuthContextNotCooled, +} from "../../src/codex/auth-context"; +import { createOpenAiSidecarCreditGuard } from "../../src/providers/openai-sidecar-credit"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearAccountNeedsReauth, isAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { clearAccountQuota, getAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { + clearCodexUpstreamHealth, clearThreadAccountMap, getCodexQuotaHealthSnapshot, recordCodexUpstreamOutcome, +} from "../../src/codex/routing"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const ID = "scoped-credit-fixture"; +const NOW = 1_800_000_000_000; +const RESET = NOW + 6 * 86_400_000; +const MODEL = "gpt-5.5"; +let home: string; +let previousHome: string | undefined, previousCodexHome: string | undefined; +let clock: ReturnType; +let originalFetch: typeof fetch; +let networkCalls: number; +let expectedNetworkCalls: number; +const cfg = (): OcxConfig => ({ + port: 10100, defaultProvider: "openai", providers: {}, codexMainAccountHardLock: false, + codexAccounts: [{ id: ID, email: "fixture@example.test", plan: "plus", isMain: false }], + codexAccountNamespaces: { subAlias: ID }, creditCodexAccountIds: [ID], + activeCodexAccountId: ID, autoSwitchThreshold: 0, upstreamFailoverThreshold: 3, +}); +function observe(age: number) { + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: RESET, + credits: { hasCredits: true, balance: 5, allowed: true, observedAt: NOW - age } }); +} +const resolve = (config: OcxConfig) => resolveCodexAuthContext(new Headers(), config, "direct", { accountId: ID, modelId: MODEL }); +function state() { + return structuredClone({ quota: getAccountQuota(ID), + account: getCodexQuotaHealthSnapshot(ID, undefined), + shared: getCodexQuotaHealthSnapshot(ID, "shared"), + reserve: getCodexQuotaHealthSnapshot(ID, "reserve") }); +} +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; previousCodexHome = process.env.CODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-credit-scoped-retry-")); + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + clock = spyOn(Date, "now").mockReturnValue(NOW); + const ok = { success: true, exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + setIcaclsRunnerForTests(() => ok); setAsyncIcaclsRunnerForTests(async () => ok); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + saveCodexAccountCredential(ID, { accessToken: "fixture-scoped-access", refreshToken: "fixture-scoped-refresh", + expiresAt: NOW + 3_600_000, chatgptAccountId: "fixture-scoped-workspace" }); + originalFetch = globalThis.fetch; networkCalls = 0; expectedNetworkCalls = 0; + globalThis.fetch = Object.assign((): never => { networkCalls++; throw new Error("Unexpected network request"); }, + { preconnect: originalFetch.preconnect }); +}); +afterEach(async () => { + try { + expect(networkCalls).toBe(expectedNetworkCalls); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + await flushConfigDirHardeningForTests(); + } finally { + globalThis.fetch = originalFetch; clock.mockRestore(); + setIcaclsRunnerForTests(null); setAsyncIcaclsRunnerForTests(null); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); + } +}); + +const cases = [ + { name: "shared reset-derived hold", modelId: MODEL, retry: 900 }, + { name: "independent Reserve hold", modelId: "gpt-reserve", retry: 300 }, + { name: "account-wide Retry-After", modelId: "gpt-reserve", retryAfter: "3600", retry: 3600 }, + { name: "expired shared hold", modelId: MODEL, age: 901_000, retry: 300 }, + { name: "shared hold recorded before a shorter account hold", modelId: MODEL, retryAfter: "600", combined: "before", retry: 900 }, + { name: "shared hold recorded after a shorter account hold", modelId: MODEL, retryAfter: "600", combined: "after", retry: 900 }, + { name: "oversized numeric Retry-After", modelId: MODEL, retryAfter: "99999999999999999999", retry: 86_400 }, + { name: "far-future Retry-After date", modelId: MODEL, retryAfter: "Fri, 31 Dec 9999 23:59:59 GMT", retry: 86_400 }, +] as const; +for (const boundary of ["resolve", "materialize", "materialize-async", "dispatch", "provider", "cooldown", "sidecar"] as const) { + for (const scenario of cases) test(`${boundary} credit refusal respects ${scenario.name}`, async () => { + const config = cfg(); + observe(0); + const context = boundary === "resolve" ? undefined : await resolve(config); + if (context) expect(context).toMatchObject({ kind: "pool", accountId: ID, quotaScope: "shared" }); + observe(360_000); + const recordedAt = NOW - ("age" in scenario ? scenario.age : 0); + const recordSharedHold = () => recordCodexUpstreamOutcome(config, ID, 429, { + modelId: MODEL, resetAt: NOW + 3_600_000, fixedAccount: true, now: NOW, + }); + if ("combined" in scenario && scenario.combined === "before") recordSharedHold(); + recordCodexUpstreamOutcome(config, ID, 429, { modelId: scenario.modelId, + resetAt: recordedAt + 3_600_000, fixedAccount: true, now: recordedAt, + ...("retryAfter" in scenario ? { retryAfter: scenario.retryAfter } : {}) }); + if ("combined" in scenario && scenario.combined === "after") recordSharedHold(); + const before = state(); + let error: unknown; + try { + if (boundary === "resolve") await resolve(config); + else if (boundary === "materialize") materializeCodexUpstreamAuth(new Headers(), context!); + else if (boundary === "materialize-async") await materializeCodexUpstreamAuthAsync(new Headers(), context!); + else if (boundary === "dispatch") createCodexAuthDispatchGuard(context!, config, MODEL)!(new Headers()); + else if (boundary === "provider") applyCodexAuthContextToProvider({ + adapter: "openai-responses", baseUrl: "https://example.test", authMode: "forward", + }, context!, "pool"); + else if (boundary === "cooldown") assertCodexAuthContextNotCooled(context!); + else createOpenAiSidecarCreditGuard(context!, config)!(); + } catch (cause) { error = cause; } + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + const refusal = error as CodexPoolAccountCreditsOffError; + expect(refusal.accountId).toBe(ID); + expect(refusal.resetAt).toBe(RESET); + const response = cooldownErrorResponse(refusal, NOW); + expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe(String(scenario.retry)); + expect(state()).toEqual(before); + expect(isAccountNeedsReauth(ID)).toBe(false); + }); +} + +test("resolve retains shared scope when credits become stale during token preparation", async () => { + const config = cfg(); + observe(0); + saveCodexAccountCredential(ID, { accessToken: "fixture-expired-access", refreshToken: "fixture-scoped-refresh", + expiresAt: NOW - 1, chatgptAccountId: "fixture-scoped-workspace" }); + expectedNetworkCalls = 1; + globalThis.fetch = Object.assign(async (input: Parameters[0], init?: RequestInit) => { + networkCalls++; + expect(String(input)).toBe("https://auth.openai.com/oauth/token"); + expect(init?.method).toBe("POST"); + observe(360_000); + recordCodexUpstreamOutcome(config, ID, 429, { modelId: MODEL, + resetAt: NOW + 3_600_000, fixedAccount: true, now: NOW }); + return Response.json({ access_token: "fixture-refreshed-access", refresh_token: "fixture-refreshed-grant", expires_in: 3600 }); + }, { preconnect: originalFetch.preconnect }) as typeof fetch; + const error = await resolve(config).catch(cause => cause); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(RESET); + expect(cooldownErrorResponse(error, NOW).headers.get("retry-after")).toBe("900"); + expect(getCodexQuotaHealthSnapshot(ID, "shared")?.cooldownUntil).toBe(NOW + 900_000); + expect(isAccountNeedsReauth(ID)).toBe(false); +}); diff --git a/tests/codex-integration/main-quota-provenance.test.ts b/tests/codex-integration/main-quota-provenance.test.ts index d8cb4b4fd65..bb227bcd716 100644 --- a/tests/codex-integration/main-quota-provenance.test.ts +++ b/tests/codex-integration/main-quota-provenance.test.ts @@ -5,6 +5,8 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { MAIN_CODEX_ACCOUNT_ID as MAIN } from "../../src/codex/account-id"; +import { isCodexAccountHeldForCredits } from "../../src/codex/account-credit-use"; +import { hasSpendableCodexCredits } from "../../src/codex/quota-types"; import { getMainAccountHardLockStatus } from "../../src/codex/main-account-hard-lock"; import { applyConfirmedMainCodexAccountTransition, @@ -341,6 +343,77 @@ describe("main policy quota durability and lifecycle", () => { }); }, COLD_SPAWN_WARMUP_HOOK_BUDGET_MS); + for (const unavailable of [true, false]) { + test(`persisted ${unavailable ? "unavailable" : "legacy"} null remains identity-bound and cannot authorize spending`, () => { + const writer = writerFor(); + const resetAt = Date.now() + 86_400_000; + setAccountQuotaFromParsed(MAIN, { weeklyPercent: 100, weeklyResetAt: resetAt, + credits: { hasCredits: true, balance: 5, observedAt: Date.now() } }, undefined, writer); + setAccountQuotaFromParsed(MAIN, unavailable ? parseUsageQuota({ credits: {} }) : { credits: null }, + undefined, writer); + // Partial headers carry the pair without supplying new spending evidence. + applyAccountQuotaFromUpstreamHeaders(MAIN, new Headers({ "x-codex-primary-used-percent": "100", + "x-codex-primary-reset-at": String(resetAt) }), + undefined, writer); + const quota = getMainPolicyQuota(); + expect(quota?.credits).toBeNull(); + expect(quota?.creditObservationUnavailable).toBe(unavailable ? true : undefined); + expect(hasSpendableCodexCredits(quota)).toBe(false); + expect(isCodexAccountHeldForCredits({ creditCodexAccountIds: [MAIN] }, MAIN, quota, "plus", Date.now())).toBe(true); + const persisted = JSON.parse(flushPersistence()); + expect(persisted.mainPolicyQuota).toEqual({ identityKey: writer.identityKey, quota }); + // Read the actual serializer output in a fresh process, without clearing or rewriting it. + const script = ` + ${QUOTA_PROVENANCE_IMPORT_PROLOGUE} + import { hasSpendableCodexCredits } from ${JSON.stringify(repoPath("src/codex/quota-types.ts"))}; + import { isCodexAccountHeldForCredits } from ${JSON.stringify(repoPath("src/codex/account-credit-use.ts"))}; + globalThis.fetch = () => { throw new Error("Unexpected network request"); }; + const before = getMainPolicyQuota(); + observeMainQuotaIdentity("fixture-main-b"); + const other = getMainPolicyQuota(); + observeMainQuotaIdentity("fixture-main-a"); + const policy = getMainPolicyQuota(); + console.log(JSON.stringify({ before, other, policy, + spendable: hasSpendableCodexCredits(policy), + held: isCodexAccountHeldForCredits({ creditCodexAccountIds: ["__main__"] }, "__main__", policy, "plus", Date.now()) })); + `; + const child = Bun.spawnSync({ + cmd: [process.execPath, "--eval", script], cwd: repoRoot(), env: process.env, + timeout: process.platform === "win32" ? SPAWN_BUDGET_MS - INTERNAL_DEADLINE_MS : 10_000, + }); + expect(child.exitCode).toBe(0); + expect(JSON.parse(child.stdout.toString())).toEqual({ before: null, other: null, policy: quota, + spendable: false, held: true }); + }, SPAWN_BUDGET_MS); + } + + test("stale main writers cannot replace unavailable evidence or attach it to a new identity", () => { + const oldA = writerFor(); + const unavailable = parseUsageQuota({ credits: {} }); + const positive = { hasCredits: true, balance: 5, observedAt: Date.now() }; + setAccountQuotaFromParsed(MAIN, unavailable, undefined, oldA); + const writerB = writerFor("fixture-main-b"); + expect(getMainPolicyQuota()).toBeNull(); + setAccountQuotaFromParsed(MAIN, { credits: positive }, undefined, writerB); + setAccountQuotaFromParsed(MAIN, unavailable, undefined, oldA); + expect(getMainPolicyQuota()?.credits).toEqual(positive); + expect(getMainPolicyQuota()?.creditObservationUnavailable).toBeUndefined(); + + const newA = writerFor(); + setAccountQuotaFromParsed(MAIN, { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000 }, undefined, newA); + setAccountQuotaFromParsed(MAIN, unavailable, undefined, newA); + const before = getMainPolicyQuota(); + expect(before).toMatchObject({ credits: null, creditObservationUnavailable: true }); + // Returning to A does not revive a writer captured before the A → B → A transition. + setAccountQuotaFromParsed(MAIN, { credits: positive }, undefined, oldA); + expect(isMainQuotaWriterLive(oldA)).toBe(false); + expect(getMainPolicyQuota()).toEqual(before); + expect(getAccountQuota(MAIN)?.credits).toBeNull(); + expect(hasSpendableCodexCredits(getMainPolicyQuota())).toBe(false); + expect(isCodexAccountHeldForCredits({ creditCodexAccountIds: [MAIN] }, MAIN, getMainPolicyQuota(), "plus", Date.now())).toBe(true); + expect(JSON.parse(flushPersistence()).mainPolicyQuota).toEqual({ identityKey: newA.identityKey, quota: before }); + }); + for (const resetAt of [undefined, 4_000_000_000]) { test(`restart beyond six hours retains ${resetAt ? "future-reset" : "missing-reset"} policy evidence only for observed A`, () => { const writer = writerFor(); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index cead7ca3353..c921078bca9 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -177,7 +177,7 @@ "codex-config-preservation.test.ts": "codex-integration", "codex-credits.test.ts": "codex-integration", "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", - "codex-pool-credit-policy.test.ts": "codex-integration", + "codex-pool-credit-policy.test.ts": "codex-integration", "codex-credit-retry-after.test.ts": "codex-integration", "codex-credit-scoped-retry.test.ts": "codex-integration", "codex-direct-credit-renewal.test.ts": "codex-integration", "codex-direct-renewal-revocation.test.ts": "codex-integration", "direct-credit-renewal-ingress.test.ts": "responses", "codex-sidecar-credit-policy.test.ts": "codex-integration", diff --git a/tests/responses/direct-credit-renewal-ingress.test.ts b/tests/responses/direct-credit-renewal-ingress.test.ts index c722ca9367b..3df9a1c9fe2 100644 --- a/tests/responses/direct-credit-renewal-ingress.test.ts +++ b/tests/responses/direct-credit-renewal-ingress.test.ts @@ -112,7 +112,7 @@ async function send() { body: JSON.stringify({ model: "subAlias/gpt-5.5", input: "fixture", stream: false, store: false }), }); const response = await handleResponses(request, config, { model: "", provider: "" }); - return { status: response.status, body: await response.json() }; + return { status: response.status, retryAfter: response.headers.get("retry-after"), body: await response.json() }; } test("same Direct namespace recovers after five-minute credit expiry without fallback or external refresh", async () => { @@ -123,6 +123,7 @@ test("same Direct namespace recovers after five-minute credit expiry without fal now += 360_001; const stale = await send(); expect(stale.status).toBe(429); + expect(stale.retryAfter).toBe("300"); expect(responseAccounts).toHaveLength(1); expect(metadataAccounts).toEqual([]); const noMain = () => { throw new Error("stored-account renewal must never access main"); };