From 9995b64a2ffaa68d9a04fc3ee2f865a9e559eff0 Mon Sep 17 00:00:00 2001 From: Caledvwlch Date: Fri, 23 Oct 2026 20:27:04 +0900 Subject: [PATCH 1/6] fix(codex): renew credit evidence for namespace-bound Direct accounts --- .../content/docs/guides/codex-integration.md | 2 + .../docs/ko/guides/codex-integration.md | 2 + .../test-layout/layout-credit-renewal.json | 4 + scripts/test-layout/layout.json | 2 +- src/codex/auth-api/pool-mode-gate.ts | 19 +- src/codex/auth-api/pool-quota-probe.ts | 15 + structure/providers/openai-accounts.md | 2 +- structure/providers/openai-tiers.md | 2 +- structure/providers/pool-credit-renewal.md | 29 +- .../codex-direct-credit-renewal.test.ts | 318 ++++++++++++++++++ .../codex-direct-renewal-revocation.test.ts | 212 ++++++++++++ tests/fixtures/test-layout-expected.json | 2 + .../direct-credit-renewal-ingress.test.ts | 140 ++++++++ 13 files changed, 740 insertions(+), 9 deletions(-) create mode 100644 scripts/test-layout/layout-credit-renewal.json create mode 100644 tests/codex-integration/codex-direct-credit-renewal.test.ts create mode 100644 tests/codex-integration/codex-direct-renewal-revocation.test.ts create mode 100644 tests/responses/direct-credit-renewal-ingress.test.ts diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index 57373b06ece..5faad6b94b7 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -1274,6 +1274,8 @@ An allowed response with primary and secondary windows can omit the optional ter This recovery uses your included plan allowance and does not require enabling **Use credits**. Partial or invalid usage data retains the previous block until recovery is confirmed. +Stored accounts with **Use credits after limit** enabled renew their credit evidence on the existing background cycle once an included usage window is exhausted and the credit observation is at least three minutes old. In **Direct** mode this also covers stored accounts explicitly bound in `codexAccountNamespaces`, such as a sub-account selector. You can keep Direct mode and separate main/sub identities; this does not enable Pool routing or refresh unbound accounts. Paused and reauthentication-required accounts are skipped. Metadata reads do not validate pending accounts or send model requests. Failed reads retain backoff and requests still need fresh spendable credits; ordinary response usage headers do not renew the credit balance timestamp. + ### Cancelling main-account device reauthentication When cancelling main-account device reauthentication, a temporary DELETE or network failure, or a response with an unknown or nonterminal status, keeps the active flow and the cancellation-failure indication so cancellation can be retried. Status polling normally continues so a login that completes can still be detected. If a retryable cancellation failure overlaps a non-2xx GET status response while the flow is `pending` or `committing`, either response order preserves cancellation retry on that same flow, with its last server-provided device code, verification URL, and phase. The GET HTTP failure still stops polling, but cancellation can be retried without starting a second login POST. A terminal `failed` response releases the flow and displays the normalized failure reason; only `succeeded` reports login success. A confirmed `cancelled` response releases the flow so a new device login can be started. A definitive HTTP 404 response with code `unknown_flow` also releases the expired flow ID so a new device login can be started, but does not report a successful login or confirmed cancellation. Late POST, GET, or DELETE responses from an earlier flow cannot change the new flow or report login success for it. diff --git a/docs-site/src/content/docs/ko/guides/codex-integration.md b/docs-site/src/content/docs/ko/guides/codex-integration.md index fb1c2370de0..ad7c94bb7d5 100644 --- a/docs-site/src/content/docs/ko/guides/codex-integration.md +++ b/docs-site/src/content/docs/ko/guides/codex-integration.md @@ -390,6 +390,8 @@ ChatGPT 계정을 추가하거나 재인증할 때 OpenCodex는 일반적으로 **Codex Set → Multi-auth**에서 **Codex Auth** 제목 줄의 **Codex 크레딧** 스위치를 켜면 메인 계정과 풀 계정의 최근 크레딧 잔액이 Week 바로 아래에 표시됩니다. 기본값은 꺼짐이며 `showCodexCredits`로 저장됩니다. 잔액은 로케일에 맞는 숫자로 표시하고, 응답에 따라 무제한 또는 초과 사용 한도 도달 안내를 표시합니다. 전체 크레딧 한도가 제공되지 않으므로 막대는 백분율이 아닌 사용 가능 상태를 나타냅니다. 스위치는 표시만 제어하며 새 로그인에는 해당 계정의 조회 결과가 필요합니다. +**Use credits after limit**을 허용한 저장 계정은 포함 사용량 한도가 소진되고 크레딧 관측이 3분 이상 지나면 기존 백그라운드 주기에서 잔액을 다시 조회합니다. **Direct** 모드에서도 `codexAccountNamespaces`로 명시적으로 연결한 서브 계정 등이 대상입니다. Direct와 메인/서브 계정 분리를 유지하며, Pool 라우팅을 켜거나 연결되지 않은 계정을 조회하지 않습니다. 일시 정지·재인증 필요 계정은 제외합니다. 메타데이터 조회는 검증 대기 계정을 검증하거나 모델 요청을 보내지 않습니다. 조회 실패 시 재시도 대기를 유지하며 요청에는 여전히 최신의 사용 가능한 크레딧 증거가 필요합니다. 일반 응답의 사용량 헤더는 크레딧 잔액 관측 시각을 갱신하지 않습니다. + 별도의 백그라운드 재검증은 기본적으로 꺼져 있습니다. Token Guardian, `openai`의 `proactive` 갱신 정책, `tokenGuardian.codexWarmupEnabled`가 필요하며 등록 검증 대기 계정은 제외합니다. ### 계정이 요청을 처리하지 못하게 된 이유 diff --git a/scripts/test-layout/layout-credit-renewal.json b/scripts/test-layout/layout-credit-renewal.json new file mode 100644 index 00000000000..ccabb377b92 --- /dev/null +++ b/scripts/test-layout/layout-credit-renewal.json @@ -0,0 +1,4 @@ +{ + "codex-direct-credit-renewal.test.ts": "codex-integration", "codex-direct-renewal-revocation.test.ts": "codex-integration", + "direct-credit-renewal-ingress.test.ts": "responses" +} diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index e2ec2449dfd..1e5bd01048c 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1,7 +1,7 @@ { "version": 1, "root": "tests", - "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json"], + "explicitShards": ["layout-clients.json", "layout-codex-collision.json", "layout-agent-assertion.json", "layout-chat.json", "layout-replay.json", "layout-adapters.json", "layout-jev.json", "layout-workbuddy.json", "layout-credit-renewal.json"], "explicit": { "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "codex-provider-routing.test.ts": "codex-integration", "config-native-reasoning-retention.test.ts": "config", "server-native-reasoning-retention-merge.test.ts": "server", "server-native-reasoning-retention.test.ts": "server", "responses-retained-reasoning-stream-recovery.test.ts": "responses", "openai-native-hosted-search-history.test.ts": "adapters/openai", "responses-native-compact-search-history.test.ts": "responses", "config-hosted-search-projection.test.ts": "config", "gcp-credential-boundary.test.ts": "adapters/google", "gcp-sa-marker.test.ts": "adapters/google", "gcp-sa-adapter.test.ts": "adapters/google", "management-gcp-credential-input.test.ts": "server", "codex-preserve-signed-in.test.ts": "codex-integration", "cli-account-preservation.test.ts": "cli", "cli-provider-list-active.test.ts": "cli", "service-state-isolation.test.ts": "service", "compaction-history-framing.test.ts": "responses", "catalog-native-max-context.test.ts": "codex-integration", "protocol-argon-thought-boundary.test.ts": "responses", "service-log.test.ts": "service", "messages-native-empty-inline.test.ts": "claude-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "bridge-terminal-refusal.test.ts": "adapters", "loop-terminal-refusal.test.ts": "images", "web-search-terminal-refusal.test.ts": "web-search", "combo-terminal-refusal.test.ts": "routing", "server-terminal-refusal-wiring.test.ts": "server", "incomplete-reason-normalization.test.ts": "responses", "advisor-retention.test.ts": "advisor", "responses-native-key-cooldown.test.ts": "responses", "responses-run-turn-key-readmission.test.ts": "responses", "qoder-spawn-readmission.test.ts": "adapters", "api-key-selection-cooldown.test.ts": "providers", "api-key-auto-pause-distribution.test.ts": "providers", "explicit-key-pause-oauth-boundary.test.ts": "providers", "server-chat-native-key-selection.test.ts": "server", "compact-key-selection.test.ts": "responses", "messages-native-key-selection.test.ts": "claude-integration", "coding-agent-windows-dispatch.test.ts": "adapters", "coding-agent-native-empty-argv.test.ts": "windows", "service-proxy-env.test.ts": "service", "openai-chat-standalone-bare-block.test.ts": "adapters/openai", "owner-registry-isolation.test.ts": "config", diff --git a/src/codex/auth-api/pool-mode-gate.ts b/src/codex/auth-api/pool-mode-gate.ts index e024473cdb9..7907a1e6c14 100644 --- a/src/codex/auth-api/pool-mode-gate.ts +++ b/src/codex/auth-api/pool-mode-gate.ts @@ -2,7 +2,7 @@ import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt } from "../ac import { nextCodexUsageQueryAt, nextQuotaQueryDelay, pruneRemovedCodexPoolUsageAccounts } from "../quota-query-backoff"; import { CODEX_PRIORITY_FAILBACK_REFRESH_MS } from "../account-priority"; import { codexQuotaHasFreshUsage } from "../quota-observation-freshness"; -import { getCodexAccountCredential, getValidCodexToken, readCodexAccountRecord } from "../account-store"; +import { getCodexAccountCredential, getValidCodexToken, isCodexAccountGenerationLive, readCodexAccountRecord } from "../account-store"; import { getAccountQuota, getMainPolicyQuota, isCompleteCodexQuotaRecoverySnapshot } from "../quota"; import { isCodexAccountPaused } from "../account-pause"; import { hasSpendableCodexCredits } from "../quota-types"; @@ -80,6 +80,13 @@ const MAIN_CREDIT_RENEWAL_AGE_MS = 3 * 60_000; /** Traffic headers renew usage, never the independent credit observation (#7377). */ function poolCreditRenewalDue(config: OcxConfig, accountId: string, now = Date.now()): boolean { + const openai = config.providers[OPENAI_CODEX_PROVIDER_ID]; + if (!openai || openai.disabled === true || !isCanonicalOpenAiForwardProvider(openai)) return false; + const mode = providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai); + // Direct may renew only explicitly bound stored accounts. Check queued work too: + // credit opt-in alone does not authorize reads for every saved login. + if (mode !== "pool" && (mode !== "direct" + || !Object.values(config.codexAccountNamespaces ?? {}).includes(accountId))) return false; const account = configuredPoolAccount(config, accountId); if (!account || !codexAccountUsesCreditsAfterLimit(config, accountId) || isCodexAccountPaused(config, accountId) || isAccountNeedsReauth(accountId)) return false; @@ -313,7 +320,9 @@ export async function primeCodexPoolQuotas( !openai || openai.disabled === true || !isCanonicalOpenAiForwardProvider(openai) - || providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai) !== "pool" + || (providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai) !== "pool" + && !(reason === "credit-renewal" + && providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai) === "direct")) ) return; if (primeInFlight) return primeInFlight; // Count attempted passes, including failed reads; incoming traffic must not flood WHAM. @@ -384,7 +393,11 @@ export async function primeCodexPoolQuotas( const quota = getAccountQuota(a.id); const bypassCachedQuota = reason === "credit-renewal" || (reason === "priority-failback" && quota !== null && !codexQuotaHasFreshUsage(quota, a.plan, Date.now(), CODEX_PRIORITY_FAILBACK_REFRESH_MS)); - result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime); + result = await fetchPoolAccountQuota(a.id, bypassCachedQuota, a.plan, getValidPoolTokenForPrime, + false, undefined, undefined, false, reason === "credit-renewal" + ? generation => isCodexAccountGenerationLive(a.id, generation) + && poolCreditRenewalDue(getRuntimeConfig(config), a.id) + : undefined); } catch (error) { // Local quota-flight saturation proves no WHAM request existed for this account. // Consume it per item so sibling workers remain inside the shared prime lifetime. diff --git a/src/codex/auth-api/pool-quota-probe.ts b/src/codex/auth-api/pool-quota-probe.ts index 55d30da198c..abd6b2d5fe8 100644 --- a/src/codex/auth-api/pool-quota-probe.ts +++ b/src/codex/auth-api/pool-quota-probe.ts @@ -88,6 +88,8 @@ export function publishQuotaDispatch(sequence: number): void { export interface PoolQuotaProbeEvidence { onDispatch?: (sequence: number) => void; + /** Optional caller authority rechecked after token preparation, including 401 replay. */ + mayDispatch?: (credentialGeneration: number) => boolean; mayPublish?: () => boolean; attempted?: NonNullable; } @@ -109,6 +111,7 @@ export function withQuotaProbeEvidence( export interface PoolQuotaRefreshFlight { state: { + dispatchGuarded?: boolean; dispatchSequence?: number; superseded?: boolean; startCredentialGeneration?: number; @@ -255,6 +258,10 @@ export async function recoverPoolQuotaFrom401(ctx: { // generation and opens a redundant flight. ctx.onCredentialGeneration?.(refreshed.generation); + if (ctx.quotaProbeEvidence.mayDispatch?.(refreshed.generation) === false) { + return { quota: getAccountQuota(accountId), needsReauth: false, credentialGeneration: refreshed.generation, quotaProbeSkipped: true }; + } + const writerGeneration = captureConfigGeneration(); const poolWriter = capturePoolQuotaWriter(accountId, refreshed); const replayRead = await fetchCodexUsage(`pool:${accountId}:${writerGeneration}:${refreshed.generation}`, { @@ -403,6 +410,9 @@ export async function fetchFreshPoolAccountQuota( const poolWriter = capturePoolQuotaWriter(accountId, { accessToken, chatgptAccountId, generation }); requestCredentialGeneration = generation; onCredentialGeneration?.(generation); + if (quotaProbeEvidence.mayDispatch?.(generation) === false) { + return { quota: getAccountQuota(accountId), needsReauth: false, credentialGeneration: generation, quotaProbeSkipped: true }; + } const admission = await fetchCodexUsage( `pool:${accountId}:${writerGeneration}:${generation}${afterDispatchSequence === undefined ? "" : `:post-reset:${afterDispatchSequence}`}`, { headers: { Authorization: `Bearer ${accessToken}`, "ChatGPT-Account-Id": chatgptAccountId }, @@ -496,6 +506,7 @@ export async function fetchPoolAccountQuota( afterDispatchSequence?: number, recoveryProbeNow?: number, explicitRefresh = false, + mayDispatch?: PoolQuotaProbeEvidence["mayDispatch"], ): Promise { const existing = getAccountQuota(accountId); if (afterDispatchSequence === undefined && !forceRefresh && existing && Date.now() - existing.updatedAt < POOL_CACHE_TTL) { @@ -527,6 +538,8 @@ export async function fetchPoolAccountQuota( const generation = flight.state.resolvedCredentialGeneration ?? flight.state.startCredentialGeneration; return !flight.state.superseded + // An ordinary read must not inherit a renewal owner's revocable authority. + && (mayDispatch !== undefined || !flight.state.dispatchGuarded) && (afterDispatchSequence === undefined || (flight.state.dispatchSequence ?? 0) > afterDispatchSequence) && generation !== undefined && isCodexAccountGenerationLive(accountId, generation); }); @@ -543,6 +556,7 @@ export async function fetchPoolAccountQuota( for (const flight of flights ?? []) flight.state.superseded = true; } const state: PoolQuotaRefreshFlight["state"] = { + dispatchGuarded: mayDispatch !== undefined, startCredentialGeneration: record?.generation, validatePending, }; @@ -554,6 +568,7 @@ export async function fetchPoolAccountQuota( getValidToken, { onDispatch: sequence => { state.dispatchSequence = sequence; }, + mayDispatch, mayPublish: () => state.superseded !== true, }, afterDispatchSequence, diff --git a/structure/providers/openai-accounts.md b/structure/providers/openai-accounts.md index f9c7131904c..8042e07cf5a 100644 --- a/structure/providers/openai-accounts.md +++ b/structure/providers/openai-accounts.md @@ -536,7 +536,7 @@ No requests means no new polling. For this reason only, stale observation proof quota-cache freshness after the existing attempt backoff. Main refresh keeps its owned lease and passive intent: cache bypass does not clear an inference reauthentication mark. Other prime reasons retain their existing cache rules except the independently scheduled [pool credit renewal](pool-credit-renewal.md). The split config schema degrades malformed optional values to -false. Exact-account and Direct routes are unchanged. +false. Priority failback leaves exact-account and Direct routes unchanged; explicitly bound Direct stored accounts remain eligible for the separate credit-renewal sweep. ## Main-account policy observations diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index caffc0dd5bf..4bf7d962bb0 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -334,7 +334,7 @@ identity retries: WHAM success and terminal 401/403 responses never set or clear this mode. Native token preparation also preserves the traffic quarantine. Other callers retain the existing auth behavior. Failed or incomplete observations never renew the credit clock; refreshing usage does not redeem reset credits or validate pending accounts through inference. -Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. Pool accounts use the separate [pool credit renewal](pool-credit-renewal.md) contract; their fresh spendable-evidence requirement is identical. +Coverage: `tests/codex-integration/main-account-credit-renewal.test.ts`. Stored accounts, including explicit namespace targets in Direct mode, use the separate [pool credit renewal](pool-credit-renewal.md) contract; their fresh spendable-evidence requirement is identical. Credit parsing, expiry, partial updates and reset-ticket separation are covered in `tests/codex-integration/codex-quota-parser-parity.test.ts`; selection and bulk-pause behavior diff --git a/structure/providers/pool-credit-renewal.md b/structure/providers/pool-credit-renewal.md index ad47b4ebef0..d3215d5b222 100644 --- a/structure/providers/pool-credit-renewal.md +++ b/structure/providers/pool-credit-renewal.md @@ -5,7 +5,10 @@ existing `codex-cooldown-recovery` after-tick hook, using `primeCodexPoolQuotas` `credit-renewal`. No extra timer or inference request is added. Only configured pool accounts opted into `creditCodexAccountIds` with a currently exhausted -included window qualify. Paused, reauthentication-required, below-limit and non-opted-in accounts +included window qualify. In Direct mode, a stored account must also be a current target of +`codexAccountNamespaces`; duplicate aliases still produce one account read. Unqualified Direct +requests do not enable general pool priming. Startup, pre-route and priority-failback reasons +remain Pool-only. Paused, reauthentication-required, below-limit and non-opted-in accounts are skipped. A credit observation younger than three minutes is retained; due or missing evidence uses the existing bounded-concurrency quota reader with cache bypass, because traffic headers can keep aggregate usage fresh while the independent credit clock expires. @@ -14,11 +17,31 @@ The existing per-credential attempt map bounds retries to at least three minutes request admission and generation/configuration publication fences remain owned by the quota reader. Only actual WHAM evidence advances the credit clock. A failed read leaves stale evidence unusable; no credit freshness limit or spending consent is relaxed. Negative evidence remains discoverable. -The eligibility check runs again before each queued account read. The pool-only reason does not -claim, reconcile or read native main; its separate renewal owner is unchanged. +The eligibility check runs again before each queued account read and after asynchronous token +preparation, immediately before WHAM (including a 401 refresh/replay). It checks the current provider, +enabled state, Direct namespace binding, consent, pause/reauth, account membership and renewal age. +The prepared credential generation must still be live; a current rotated generation is accepted. +A skipped first dispatch earns no attempt marker or query backoff. Ordinary metadata readers keep +their own authority and do not join a renewal flight whose dispatch can still be revoked. +This stored-account renewal reason does not +claim, reconcile or read native main; its separate renewal owner is unchanged. In-flight metadata +remains owned by the dispatched credential even if its namespace is rebound. Revoking spending +consent still blocks auth admission after an already dispatched metadata read completes. + +All prime reasons share one in-flight pass. A renewal joining an older startup pass can leave due +credits untouched until the next sweep; it does not start a competing pass. Failures retain the +existing query backoff, and this contract does not promise uninterrupted credit availability. Regression coverage: `tests/codex-integration/codex-quota-prime.test.ts` exercises pre-expiry and expired evidence, admission after a fresh read, the sweep entry, no native-main work, exclusions and failed-read backoff. Tests use fake metadata and private homes, not paid inference. +`tests/codex-integration/codex-direct-credit-renewal.test.ts` adds schema-valid Direct bindings, +the registered sweep callback, real auth admission, coalescing, next-sweep recovery, queued policy +revocation and credential-replacement fencing. `tests/responses/direct-credit-renewal-ingress.test.ts` +checks fresh success, expired local refusal, metadata renewal and terminal success on the same +stored account through the Responses ingress, with fake upstream HTTP only. +`tests/codex-integration/codex-direct-renewal-revocation.test.ts` suspends token preparation to +verify policy revocation, credential deletion/replacement, current-generation acceptance, replay +revocation and ordinary metadata isolation through the real primer and quota reader. > Decision record: [ADR-7377](../decisions/ADR-7377-pool-credit-renewal.md) diff --git a/tests/codex-integration/codex-direct-credit-renewal.test.ts b/tests/codex-integration/codex-direct-credit-renewal.test.ts new file mode 100644 index 00000000000..e1176862faa --- /dev/null +++ b/tests/codex-integration/codex-direct-credit-renewal.test.ts @@ -0,0 +1,318 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { configSchema } from "../../src/config/schema/config-schema"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import { codexAccountNamespaceForModel } from "../../src/codex/account-namespace-match"; +import { CodexPoolAccountCreditsOffError, materializeCodexUpstreamAuth, resolveCodexAuthContext } from "../../src/codex/auth-context"; +import { isCodexAccountHeldForCredits } from "../../src/codex/account-credit-use"; +import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { readCodexAccountRecord, saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearCodexQuotaPrimeState, primeCodexPoolQuotas, registerCodexCooldownRecoveryProbeWorker } from "../../src/codex/auth-api/pool-mode-gate"; +import { clearAccountQuota, getAccountQuota, setAccountQuotaFromParsed, updateAccountQuota } from "../../src/codex/quota"; +import { resetQuotaQueryBackoffForTests } from "../../src/codex/quota-query-backoff"; +import { settleBackgroundWork } from "../../src/lib/background-work"; +import * as sweeper from "../../src/lib/state-store-sweeper"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +// Real storage, config schema, primer, credential resolution, quota query and parser; +// only the upstream HTTP boundary and wall clock are replaced. No inference is permitted. +describe("Direct explicit-account credit renewal", () => { + let home: string; + let priorHome: string | undefined; + let codexHome: IsolatedCodexHome; + let clock: ReturnType; + let originalFetch: typeof fetch; + let config: OcxConfig; + let now: number; + let calls: string[]; + let unexpected: string[]; + let reply: (accountId: string) => Response | Promise; + let nativeReads: number; + + function saveCredential(id: string, suffix = "") { + saveCodexAccountCredential(id, { + accessToken: `fixture-access-${id}${suffix}`, refreshToken: `fixture-refresh-${id}${suffix}`, + chatgptAccountId: `fixture-account-${id}`, expiresAt: now + 3_600_000, + }); + } + function seed(id: string, age = 360_000, percent = 100) { + config.codexAccounts!.push({ id, isMain: false, plan: "plus", email: `${id}@example.test` }); + saveCredential(id); + setAccountQuotaFromParsed(id, { + weeklyPercent: percent, weeklyResetAt: now + 604_800_000, + credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now - age }, + }); + } + function wham(credits: unknown = { has_credits: true, balance: 7 }, spendControl: unknown = { reached: false }) { + return Response.json({ plan_type: "plus", rate_limit: { + secondary_window: { used_percent: 100, reset_at: (now + 604_800_000) / 1000 }, + }, ...(credits === undefined ? {} : { credits }), spend_control: spendControl }); + } + function held(id = "stored-a") { + return isCodexAccountHeldForCredits(config, id, getAccountQuota(id), "plus", now); + } + function prime(reason = "credit-renewal") { + const noMain = () => { nativeReads++; throw new Error("renewal must not access native main"); }; + return primeCodexPoolQuotas(config, reason, { + reconcileMainAccount: noMain, readMainTokens: noMain, fetchMainInfo: noMain, + }); + } + function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; + } + function registeredTick() { + let tick!: () => void; + const registration = spyOn(sweeper, "registerStateSweepAfterTick").mockImplementation(entry => { + expect(entry.name).toBe("codex-cooldown-recovery"); tick = entry.afterTick; + return () => {}; + }); + try { registerCodexCooldownRecoveryProbeWorker(config); } + finally { registration.mockRestore(); } + expect(tick).toBeFunction(); + return tick; + } + + beforeEach(() => { + priorHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-direct-credit-")); + process.env.OPENCODEX_HOME = home; + codexHome = installIsolatedCodexHome("ocx-direct-credit-native-"); + writeFileSync(join(codexHome.path, "auth.json"), JSON.stringify({ + tokens: { access_token: "fixture-native-main", account_id: "fixture-native-account" }, + })); + now = Date.now(); + clock = spyOn(Date, "now").mockReturnValue(now); + clearCodexQuotaPrimeState(); clearAccountQuota(); resetQuotaQueryBackoffForTests(); + calls = []; unexpected = []; nativeReads = 0; + config = { + port: 10100, defaultProvider: "openai", providers: { + openai: { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", codexAccountMode: "direct" }, + }, + codexAccounts: [], codexAccountNamespaces: { subAlias: "stored-a", mainAlias: "@main" }, + creditCodexAccountIds: ["stored-a", "stored-b"], + } as OcxConfig; + seed("stored-a"); seed("stored-b"); + // Prevent the old source-slice fixture's namespace/account-id collision from recurring. + expect(configSchema.safeParse(config).success).toBe(true); + originalFetch = globalThis.fetch; + reply = () => wham(); + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const headers = new Headers(init?.headers); + const account = headers.get("chatgpt-account-id") ?? ""; + const id = account.replace(/^fixture-account-/, ""); + if (url !== "https://chatgpt.com/backend-api/wham/usage" || (init?.method ?? "GET") !== "GET" + || !account.startsWith("fixture-account-")) { + unexpected.push("non-fixture metadata request"); + throw new Error("only fixture WHAM requests are permitted"); + } + expect(headers.get("authorization")).toBe(`Bearer ${readCodexAccountRecord(id)!.credential!.accessToken}`); + calls.push(id); + return reply(id); + }) as typeof fetch; + }); + + afterEach(async () => { + const drained = await settleBackgroundWork({ timeoutMs: 1000 }); + globalThis.fetch = originalFetch; + clock.mockRestore(); + clearAccountNeedsReauth("stored-a"); + clearCodexQuotaPrimeState(); clearAccountQuota(); resetQuotaQueryBackoffForTests(); + await flushConfigDirHardeningForTests(); + if (priorHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = priorHome; + codexHome.restoreEnvironment(); + if (drained.settled) { removeTreeWithRetry(home); codexHome.removeAfterDrain(); } + expect(drained.settled).toBe(true); + expect(unexpected).toEqual([]); + expect(nativeReads).toBe(0); + }); + + test.each([180_000, 360_000])("renews at %d ms through real WHAM parsing despite fresh usage", async age => { + setAccountQuotaFromParsed("stored-a", { credits: { + hasCredits: true, balance: 5, allowed: true, observedAt: now - age, + } }); + updateAccountQuota("stored-a", 100); + expect(getAccountQuota("stored-a")!.updatedAt).toBe(now); + expect(getAccountQuota("stored-a")!.credits!.observedAt).toBe(now - age); + expect(held()).toBe(age > 300_000); + expect(codexAccountNamespaceForModel(config.codexAccountNamespaces, "subAlias/gpt-5.5")).toBe("subAlias"); + const before = structuredClone(config); + await prime(); + expect(calls).toEqual(["stored-a"]); + expect(getAccountQuota("stored-a")!.credits).toMatchObject({ balance: 7, observedAt: now, allowed: true }); + expect(held()).toBe(false); + expect(getAccountQuota("stored-b")!.credits!.observedAt).toBe(now - 360_000); + expect(config).toEqual(before); + }); + + test("deduplicates aliases and coalesces concurrent renewals", async () => { + config.codexAccountNamespaces!.anotherAlias = "stored-a"; + expect(configSchema.safeParse(config).success).toBe(true); + const entered = deferred(), release = deferred(); + reply = async () => { entered.resolve(); await release.promise; return wham(); }; + const first = prime(), second = prime(); + try { await Promise.race([entered.promise, first]); expect(calls).toEqual(["stored-a"]); } + finally { release.resolve(); await Promise.all([first, second]); } + await prime(); + expect(calls).toEqual(["stored-a"]); + expect(held()).toBe(false); + }); + + test("the registered sweep releases real Direct auth only for the namespace's stored credential", async () => { + const namespace = codexAccountNamespaceForModel(config.codexAccountNamespaces, "subAlias/gpt-5.5")!; + const caller = new Headers({ authorization: "Bearer fixture-caller-main", "chatgpt-account-id": "fixture-caller-account" }); + const resolve = () => resolveCodexAuthContext(caller, config, "direct", { + accountId: config.codexAccountNamespaces![namespace], modelId: "gpt-5.5", + }); + await expect(resolve()).rejects.toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(calls).toEqual([]); + registeredTick()(); + // Join the work started by the actual registered callback, rather than sleeping for a timer. + expect((await settleBackgroundWork({ timeoutMs: 1000 })).settled).toBe(true); + expect(calls).toEqual(["stored-a"]); + const context = await resolve(); + expect(context.kind).toBe("pool"); expect(context.accountId).toBe("stored-a"); + const auth = materializeCodexUpstreamAuth(caller, context); + expect(auth.get("authorization")).toBe("Bearer fixture-access-stored-a"); + expect(auth.get("chatgpt-account-id")).toBe("fixture-account-stored-a"); + expect(caller.get("authorization")).toBe("Bearer fixture-caller-main"); + }); + + test("renewal joining a pending startup prime is retried by the next sweep", async () => { + config.providers.openai!.codexAccountMode = "pool"; + clock.mockReturnValue(now + 360_000); + // Startup has only B to refresh: A's usage is fresh but its credit evidence is not. + updateAccountQuota("stored-a", 100); + const entered = deferred(), release = deferred(); + reply = async () => { entered.resolve(); await release.promise; return wham(); }; + const running = primeCodexPoolQuotas(config, "startup", { + reconcileMainAccount: () => false, readMainTokens: () => null, + }); + const tick = registeredTick(); + try { + await Promise.race([entered.promise, running]); expect(calls).toEqual(["stored-b"]); + config.providers.openai!.codexAccountMode = "direct"; + tick(); expect(calls).toEqual(["stored-b"]); + } finally { release.resolve(); await running; } + expect(getAccountQuota("stored-a")!.credits!.observedAt).toBe(now - 360_000); + reply = () => wham(); tick(); + expect((await settleBackgroundWork({ timeoutMs: 1000 })).settled).toBe(true); + expect(calls).toEqual(["stored-b", "stored-a"]); + }); + + test.each(["binding", "consent", "pause", "reauth", "provider", "removal"])("rechecks queued %s revocation", async revoked => { + const ids = ["stored-a", "stored-b", "stored-c", "stored-d", "queued-account"]; + for (const id of ids.slice(2)) seed(id); + config.creditCodexAccountIds = ids; + config.codexAccountNamespaces = Object.fromEntries(ids.map((id, i) => [`alias${i}`, id])); + expect(configSchema.safeParse(config).success).toBe(true); + const entered = deferred(), release = deferred(); + reply = async () => { if (calls.length === 4) entered.resolve(); await release.promise; return wham(); }; + const running = prime(); + try { + await Promise.race([entered.promise, running]); + expect(calls).toHaveLength(4); + if (revoked === "binding") delete config.codexAccountNamespaces.alias4; + if (revoked === "consent") config.creditCodexAccountIds = ids.slice(0, 4); + if (revoked === "pause") config.pausedCodexAccountIds = ["queued-account"]; + if (revoked === "reauth") markAccountNeedsReauth("queued-account"); + if (revoked === "provider") config.providers.openai!.disabled = true; + if (revoked === "removal") config.codexAccounts = config.codexAccounts!.filter(a => a.id !== "queued-account"); + } finally { release.resolve(); await running; clearAccountNeedsReauth("queued-account"); } + expect(calls).toEqual(ids.slice(0, 4)); + }); + + test("a replaced credential rejects late credit publication and does not inherit attempt backoff", async () => { + const entered = deferred(), release = deferred(); + reply = async () => { entered.resolve(); await release.promise; return wham(); }; + const running = prime(); + try { + await Promise.race([entered.promise, running]); expect(calls).toEqual(["stored-a"]); + saveCredential("stored-a", "-replacement"); + setAccountQuotaFromParsed("stored-a", { credits: { + hasCredits: true, balance: 9, allowed: true, observedAt: now - 360_000, + } }); + } finally { release.resolve(); await running; } + expect(getAccountQuota("stored-a")!.credits).toMatchObject({ balance: 9, observedAt: now - 360_000 }); + expect(held()).toBe(true); + reply = () => wham(); + await prime(); + expect(calls).toEqual(["stored-a", "stored-a"]); + expect(held()).toBe(false); + }); + + test("in-flight rebinding cannot publish the old account's credits onto its new target or authorize opted-out spend", async () => { + const entered = deferred(), release = deferred(); + reply = async () => { entered.resolve(); await release.promise; return wham(); }; + const running = prime(); + try { + await Promise.race([entered.promise, running]); expect(calls).toEqual(["stored-a"]); + config.codexAccountNamespaces!.subAlias = "stored-b"; + config.creditCodexAccountIds = ["stored-b"]; + } finally { release.resolve(); await running; } + expect(getAccountQuota("stored-b")!.credits!.observedAt).toBe(now - 360_000); + expect(held()).toBe(true); + reply = () => wham(); await prime(); + expect(calls).toEqual(["stored-a", "stored-b"]); + }); + + test.each([503, 429])("failed WHAM %d is paced without renewing the credit clock", async status => { + reply = () => new Response(null, { status, headers: { "Retry-After": "600" } }); + await prime(); await prime(); + expect(calls).toEqual(["stored-a"]); + expect(getAccountQuota("stored-a")!.credits!.observedAt).toBe(now - 360_000); + clock.mockReturnValue(now + 180_000); + await prime(); expect(calls).toEqual(["stored-a"]); // query reader's backoff still wins + expect(held()).toBe(true); + }); + + test.each([ + { has_credits: true, balance: 0 }, { has_credits: false, balance: 7 }, + { has_credits: true, balance: 7, overage_limit_reached: true }, null, "invalid", + ])("negative WHAM credits %j never release admission", async credits => { + reply = () => wham(credits); + await prime(); expect(calls).toEqual(["stored-a"]); expect(held()).toBe(true); + }); + test("spend-control denial overrides positive funds", async () => { + reply = () => wham({ has_credits: true, balance: 7 }, { reached: true }); + await prime(); expect(calls).toEqual(["stored-a"]); expect(held()).toBe(true); + }); + test("passive renewal does not validate a pending credential or send inference", async () => { + saveCodexAccountCredential("stored-a", readCodexAccountRecord("stored-a")!.credential!, { validationPending: true }); + await prime(); expect(calls).toEqual(["stored-a"]); + expect(readCodexAccountRecord("stored-a")!.codexValidationPending).toBe(true); + }); + test("Pool keeps renewal of all eligible stored accounts", async () => { + config.providers.openai!.codexAccountMode = "pool"; + await prime(); expect(calls).toEqual(["stored-a", "stored-b"]); + }); + test.each(["startup", "pre-route", "priority-failback"])("Direct %s still does not prime", async reason => { + await prime(reason); expect(calls).toEqual([]); + }); + test.each(["plain", "disabled", "api", "below-limit", "young", "paused", "opt-out", "reauth"])("skips %s", async condition => { + if (condition === "plain") config.codexAccountNamespaces = {}; + if (condition === "disabled") config.providers.openai!.disabled = true; + if (condition === "api") { + config.providers = { "openai-apikey": { adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", authMode: "key", apiKey: "fixture-api-key" } }; + config.defaultProvider = "openai-apikey"; + expect(configSchema.safeParse(config).success).toBe(true); + } + if (condition === "below-limit") updateAccountQuota("stored-a", 2); + if (condition === "young") setAccountQuotaFromParsed("stored-a", { credits: { + hasCredits: true, balance: 5, allowed: true, observedAt: now - 179_999, + } }); + if (condition === "paused") config.pausedCodexAccountIds = ["stored-a"]; + if (condition === "opt-out") config.creditCodexAccountIds = []; + if (condition === "reauth") markAccountNeedsReauth("stored-a"); + await prime(); expect(calls).toEqual([]); + }); +}); diff --git a/tests/codex-integration/codex-direct-renewal-revocation.test.ts b/tests/codex-integration/codex-direct-renewal-revocation.test.ts new file mode 100644 index 00000000000..6f3dec6284d --- /dev/null +++ b/tests/codex-integration/codex-direct-renewal-revocation.test.ts @@ -0,0 +1,212 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import { clearAccountNeedsReauth, markAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { getValidCodexToken, readCodexAccountRecord, removeCodexAccountCredential, saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearCodexQuotaPrimeState, primeCodexPoolQuotas, setCodexPoolQuotaTokenResolverForTests } from "../../src/codex/auth-api/pool-mode-gate"; +import { fetchPoolAccountQuota } from "../../src/codex/auth-api/pool-quota-probe"; +import { clearAccountQuota, getAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { resetQuotaQueryBackoffForTests } from "../../src/codex/quota-query-backoff"; +import { settleBackgroundWork } from "../../src/lib/background-work"; +import type { OcxConfig } from "../../src/types"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +function deferred() { + let resolve!: () => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +// Real primer, store, WHAM reader and parser. Only token timing and external HTTP +// are controlled; every unexpected HTTP request fails closed inside the fixture. +describe("credit renewal rechecks authority after token preparation", () => { + const id = "renewal-fixture"; + let home: string, priorHome: string | undefined, nativeHome: IsolatedCodexHome; + let config: OcxConfig, observedAt: number; + let originalFetch: typeof fetch; + let calls: { account: string | null; bearer: string | null }[]; + let unexpected: string[]; + let reply: () => Response | Promise; + let refreshReply: () => Response | Promise; + + function save(suffix = "") { + saveCodexAccountCredential(id, { + accessToken: `fixture-access${suffix}`, refreshToken: `fixture-refresh${suffix}`, + chatgptAccountId: "fixture-workspace", expiresAt: Date.now() + 3_600_000, + }); + } + function wham() { + return Response.json({ plan_type: "plus", rate_limit: { + secondary_window: { used_percent: 100, reset_at: (Date.now() + 604_800_000) / 1000 }, + }, credits: { has_credits: true, balance: 7 } }); + } + function revoke(kind: string) { + if (kind === "binding") config.codexAccountNamespaces = {}; + if (kind === "rebind") config.codexAccountNamespaces = { subAlias: "other-fixture" }; + if (kind === "consent") config.creditCodexAccountIds = []; + if (kind === "pause") config.pausedCodexAccountIds = [id]; + if (kind === "provider") config.providers.openai!.disabled = true; + if (kind === "provider-replacement") config.providers.openai!.baseUrl = "https://example.test/v1"; + if (kind === "reauth") markAccountNeedsReauth(id); + if (kind === "removal") config.codexAccounts = []; + if (kind === "replacement") save("-replacement"); + if (kind === "deletion") removeCodexAccountCredential(id); + } + function prime() { return primeCodexPoolQuotas(config, "credit-renewal"); } + function delayedToken() { + const entered = deferred(), release = deferred(); + const resolver: typeof getValidCodexToken = async accountId => { + const token = await getValidCodexToken(accountId); + entered.resolve(); + await release.promise; + return token; + }; + return { entered, release, resolver }; + } + + beforeEach(() => { + priorHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-renewal-revocation-")); + process.env.OPENCODEX_HOME = home; + nativeHome = installIsolatedCodexHome("ocx-renewal-revocation-native-"); + clearCodexQuotaPrimeState(); clearAccountQuota(); resetQuotaQueryBackoffForTests(); + config = { + port: 10100, defaultProvider: "openai", providers: { + openai: { adapter: "openai-responses", baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", codexAccountMode: "direct" }, + }, + codexAccounts: [{ id, isMain: false, plan: "plus", email: "fixture@example.test" }], + codexAccountNamespaces: { subAlias: id }, creditCodexAccountIds: [id], + } as OcxConfig; + save(); observedAt = Date.now() - 360_000; + setAccountQuotaFromParsed(id, { weeklyPercent: 100, weeklyResetAt: Date.now() + 604_800_000, + credits: { hasCredits: true, balance: 5, observedAt } }); + calls = []; unexpected = []; reply = wham; + refreshReply = () => { throw new Error("unexpected fixture token refresh"); }; + originalFetch = globalThis.fetch; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + if (url === "https://auth.openai.com/oauth/token") return refreshReply(); + if (url !== "https://chatgpt.com/backend-api/wham/usage" || (init?.method ?? "GET") !== "GET") { + unexpected.push(url); throw new Error("non-fixture HTTP is forbidden"); + } + const headers = new Headers(init?.headers); + calls.push({ account: headers.get("chatgpt-account-id"), bearer: headers.get("authorization") }); + return reply(); + }) as typeof fetch; + }); + + afterEach(async () => { + const drained = await settleBackgroundWork({ timeoutMs: 1000 }); + clearAccountNeedsReauth(id); + clearCodexQuotaPrimeState(); clearAccountQuota(); resetQuotaQueryBackoffForTests(); + await flushConfigDirHardeningForTests(); + globalThis.fetch = originalFetch; + if (priorHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = priorHome; + nativeHome.restoreEnvironment(); + if (drained.settled) { removeTreeWithRetry(home); nativeHome.removeAfterDrain(); } + expect(drained.settled).toBe(true); expect(unexpected).toEqual([]); + }); + + test.each(["binding", "rebind", "consent", "pause", "provider", "provider-replacement", "reauth", "removal", "replacement", "deletion"])( + "does not dispatch a prepared token after %s", async kind => { + const delay = delayedToken(); + const restore = setCodexPoolQuotaTokenResolverForTests(delay.resolver); + const running = prime(); + try { + await Promise.race([delay.entered.promise, running]); + expect(calls).toEqual([]); + revoke(kind); + } finally { delay.release.resolve(); await running; restore(); } + expect(calls).toEqual([]); + expect(getAccountQuota(id)?.credits?.observedAt).not.toBeGreaterThan(observedAt); + }, + ); + + test.each(["consent", "pause", "provider", "removal"])("Pool renewal also rechecks %s after token preparation", async kind => { + config.providers.openai!.codexAccountMode = "pool"; + const delay = delayedToken(); + const restore = setCodexPoolQuotaTokenResolverForTests(delay.resolver); + const running = prime(); + try { + await Promise.race([delay.entered.promise, running]); expect(calls).toEqual([]); revoke(kind); + } finally { delay.release.resolve(); await running; restore(); } + expect(calls).toEqual([]); + }); + + test("a skipped read does not consume the next eligible renewal's attempt budget", async () => { + const delay = delayedToken(); + const restore = setCodexPoolQuotaTokenResolverForTests(delay.resolver); + const running = prime(); + try { + await Promise.race([delay.entered.promise, running]); expect(calls).toEqual([]); revoke("binding"); + } finally { delay.release.resolve(); await running; restore(); } + expect(calls).toEqual([]); + config.codexAccountNamespaces = { subAlias: id }; + await prime(); + expect(calls).toEqual([{ account: "fixture-workspace", bearer: "Bearer fixture-access" }]); + expect(getAccountQuota(id)?.credits?.balance).toBe(7); + }); + + test("accepts the current generation returned by token preparation", async () => { + const entered = deferred(), release = deferred(); + const before = readCodexAccountRecord(id)!.generation; + const restore = setCodexPoolQuotaTokenResolverForTests(async accountId => { + entered.resolve(); await release.promise; + return getValidCodexToken(accountId); + }); + const running = prime(); + try { + await Promise.race([entered.promise, running]); expect(calls).toEqual([]); save("-rotated"); + } finally { release.resolve(); await running; restore(); } + expect(readCodexAccountRecord(id)!.generation).toBeGreaterThan(before); + expect(calls).toEqual([{ account: "fixture-workspace", bearer: "Bearer fixture-access-rotated" }]); + expect(getAccountQuota(id)?.credits?.balance).toBe(7); + }); + + test.each(["binding", "consent", "pause", "provider"])("401 replay rechecks %s after real token refresh", async kind => { + const entered = deferred(), release = deferred(); + const before = readCodexAccountRecord(id)!.generation; + reply = () => calls.length === 1 ? Response.json({ error: { code: "transient_edge_rejection" } }, { status: 401 }) : wham(); + refreshReply = async () => { + entered.resolve(); await release.promise; + return Response.json({ access_token: "fixture-rotated", refresh_token: "fixture-refresh-rotated", expires_in: 3600 }); + }; + const running = prime(); + try { + await Promise.race([entered.promise, running]); expect(calls).toHaveLength(1); revoke(kind); + } finally { release.resolve(); await running; } + expect(readCodexAccountRecord(id)!.generation).toBeGreaterThan(before); + expect(calls).toEqual([{ account: "fixture-workspace", bearer: "Bearer fixture-access" }]); + expect(getAccountQuota(id)?.credits?.observedAt).toBe(observedAt); + }); + + test("ordinary metadata reads keep their independent authority", async () => { + const delay = delayedToken(); + const running = fetchPoolAccountQuota(id, true, "plus", delay.resolver); + try { + await Promise.race([delay.entered.promise, running]); expect(calls).toEqual([]); + for (const kind of ["binding", "consent", "pause", "provider"]) revoke(kind); + } finally { delay.release.resolve(); await running; } + expect(calls).toEqual([{ account: "fixture-workspace", bearer: "Bearer fixture-access" }]); + expect(getAccountQuota(id)?.credits?.balance).toBe(7); + }); + + test("ordinary metadata reads do not inherit a pending renewal's revocable authority", async () => { + const delay = delayedToken(); + const restore = setCodexPoolQuotaTokenResolverForTests(delay.resolver); + const running = prime(); + let ordinary: ReturnType | undefined; + try { + await Promise.race([delay.entered.promise, running]); expect(calls).toEqual([]); + ordinary = fetchPoolAccountQuota(id, true, "plus"); + revoke("consent"); + } finally { delay.release.resolve(); await Promise.all([running, ordinary]); restore(); } + expect(calls).toEqual([{ account: "fixture-workspace", bearer: "Bearer fixture-access" }]); + expect((await ordinary)?.freshQuota?.credits?.balance).toBe(7); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index fed4bb7ebc7..df9c4ef9ced 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -171,6 +171,8 @@ "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", "codex-pool-credit-policy.test.ts": "codex-integration", + "codex-direct-credit-renewal.test.ts": "codex-integration", "codex-direct-renewal-revocation.test.ts": "codex-integration", + "direct-credit-renewal-ingress.test.ts": "responses", "codex-sidecar-credit-policy.test.ts": "codex-integration", "service-runtime-preflight.test.ts": "service", "service-desktop-runtime-preflight.test.ts": "service", "codex-credits-after-limit-main.test.ts": "codex-integration", "service-desktop-login-registration.test.ts": "service", "service-desktop-startup-health.test.ts": "service", "service-desktop-startup-linux.test.ts": "service", diff --git a/tests/responses/direct-credit-renewal-ingress.test.ts b/tests/responses/direct-credit-renewal-ingress.test.ts new file mode 100644 index 00000000000..c722ca9367b --- /dev/null +++ b/tests/responses/direct-credit-renewal-ingress.test.ts @@ -0,0 +1,140 @@ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { configSchema } from "../../src/config/schema/config-schema"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearCodexQuotaPrimeState, primeCodexPoolQuotas } from "../../src/codex/auth-api/pool-mode-gate"; +import { clearAccountQuota, getAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { resetQuotaQueryBackoffForTests } from "../../src/codex/quota-query-backoff"; +import { clearCodexUpstreamHealth, clearThreadAccountMap } from "../../src/codex/routing"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { handleResponses } from "../../src/server/responses"; +import { codexWsPool } from "../../src/server/responses/codex-ws-pool"; +import { settleBackgroundWork } from "../../src/lib/background-work"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; +import { completedPayload, drainCompactionResponseState, installCompactionRoutingAclFixture, sseResponse } from "../helpers/compaction-routing-fixtures"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +// Real ingress, namespace resolution, auth admission, WHAM parser and terminal-response +// delivery. All upstream I/O uses synthetic credentials and an allowlisted fake transport. +installCompactionRoutingAclFixture(); +let previousHome: string | undefined; +let home: string; +let codexHome: IsolatedCodexHome; +let releaseSpendHome: () => void; +let clock: ReturnType; +let now: number; +let config: OcxConfig; +let realFetch: typeof fetch; +let realWebSocket: typeof WebSocket; +let metadataAccounts: string[]; +let responseAccounts: string[]; +let unexpected: string[]; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-direct-credit-ingress-")); + process.env.OPENCODEX_HOME = home; + codexHome = installIsolatedCodexHome("ocx-credit-ingress-native-"); + releaseSpendHome = acquireOwnedSpendHome(); + clearAccountQuota(); clearCodexQuotaPrimeState(); resetQuotaQueryBackoffForTests(); + clearThreadAccountMap(); clearCodexUpstreamHealth(); clearPoolRotationState(); + now = Date.now(); + clock = spyOn(Date, "now").mockImplementation(() => now); + config = { + port: 10100, defaultProvider: "openai", providers: { openai: { + adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", + baseUrl: "https://chatgpt.com/backend-api/codex", + } }, + codexAccounts: [ + { id: "stored-a", email: "a@example.test", isMain: false, plan: "plus" }, + { id: "stored-b", email: "b@example.test", isMain: false, plan: "plus" }, + ], + codexAccountNamespaces: { subAlias: "stored-a", mainAlias: "@main" }, + creditCodexAccountIds: ["stored-a", "stored-b"], + } as OcxConfig; + expect(configSchema.safeParse(config).success).toBe(true); + for (const id of ["stored-a", "stored-b"]) { + saveCodexAccountCredential(id, { + accessToken: `fixture-access-${id}`, refreshToken: `fixture-refresh-${id}`, + expiresAt: now + 3_600_000, chatgptAccountId: `fixture-account-${id}`, + }); + setAccountQuotaFromParsed(id, { weeklyPercent: 100, weeklyResetAt: now + 604_800_000, + credits: { hasCredits: true, balance: 5, allowed: true, observedAt: now } }); + } + metadataAccounts = []; responseAccounts = []; unexpected = []; + realFetch = globalThis.fetch; realWebSocket = globalThis.WebSocket; + globalThis.WebSocket = class { + constructor() { unexpected.push("WebSocket"); throw new Error("buffered fixture must use HTTP"); } + } as unknown as typeof WebSocket; + globalThis.fetch = (async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const headers = new Headers(init?.headers); + const account = headers.get("chatgpt-account-id") ?? ""; + expect(account).toBe("fixture-account-stored-a"); + expect(headers.get("authorization")).toBe("Bearer fixture-access-stored-a"); + if (url === "https://chatgpt.com/backend-api/wham/usage" && (init?.method ?? "GET") === "GET") { + metadataAccounts.push(account); + return Response.json({ plan_type: "plus", rate_limit: { secondary_window: { + used_percent: 100, reset_at: (now + 604_800_000) / 1000, + } }, credits: { has_credits: true, balance: 5 }, spend_control: { reached: false } }); + } + if (url === "https://chatgpt.com/backend-api/codex/responses" && init?.method === "POST") { + responseAccounts.push(account); + return sseResponse([{ type: "response.completed", response: completedPayload("fixture completed") }]); + } + unexpected.push("unexpected HTTP destination"); + throw new Error("only the fixture metadata and Responses endpoints are permitted"); + }) as typeof fetch; +}); + +afterEach(async () => { + await drainCompactionResponseState(); + const drained = await settleBackgroundWork({ timeoutMs: 2000 }); + codexWsPool.dispose(); releaseSpendHome(); + globalThis.fetch = realFetch; globalThis.WebSocket = realWebSocket; clock.mockRestore(); + clearAccountQuota(); clearCodexQuotaPrimeState(); resetQuotaQueryBackoffForTests(); + clearThreadAccountMap(); clearCodexUpstreamHealth(); clearPoolRotationState(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome.restoreEnvironment(); + if (drained.settled) { removeTreeWithRetry(home); codexHome.removeAfterDrain(); } + expect(drained.settled).toBe(true); + expect(unexpected).toEqual([]); +}); + +async function send() { + const request = new Request("http://localhost/v1/responses", { method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "subAlias/gpt-5.5", input: "fixture", stream: false, store: false }), + }); + const response = await handleResponses(request, config, { model: "", provider: "" }); + return { status: response.status, body: await response.json() }; +} + +test("same Direct namespace recovers after five-minute credit expiry without fallback or external refresh", async () => { + const original = structuredClone(config); + const first = await send(); + expect(first.status).toBe(200); + expect(first.body.status).toBe("completed"); + now += 360_001; + const stale = await send(); + expect(stale.status).toBe(429); + expect(responseAccounts).toHaveLength(1); + expect(metadataAccounts).toEqual([]); + const noMain = () => { throw new Error("stored-account renewal must never access main"); }; + await primeCodexPoolQuotas(config, "credit-renewal", { + reconcileMainAccount: noMain, readMainTokens: noMain, fetchMainInfo: noMain, + }); + const recovered = await send(); + expect(recovered.status).toBe(200); + expect(recovered.body.status).toBe("completed"); + expect(recovered.body.output[0].content[0].text).toBe("fixture completed"); + expect(metadataAccounts).toEqual(["fixture-account-stored-a"]); + expect(responseAccounts).toEqual(["fixture-account-stored-a", "fixture-account-stored-a"]); + expect(getAccountQuota("stored-b")!.credits!.observedAt).toBe(now - 360_001); + expect(config).toEqual(original); +}); From a74dbb3f65f21edab44252ceb032729bc08790e1 Mon Sep 17 00:00:00 2001 From: Caledvwlch Date: Fri, 23 Oct 2026 20:27:43 +0900 Subject: [PATCH 2/6] fix(codex): recover credit retry advice without weakening account boundaries --- .../docs/reference/codex-retry-diagnostics.md | 19 ++ scripts/test-layout/layout.json | 2 +- src/codex/auth-context.ts | 28 ++- src/codex/pool-credit-policy.ts | 23 +- src/codex/quota-types.ts | 3 + src/codex/quota.ts | 17 +- src/providers/openai-sidecar-credit.ts | 4 +- structure/providers/openai-tiers.md | 7 + .../codex-credit-retry-after.test.ts | 224 ++++++++++++++++++ .../codex-credit-scoped-retry.test.ts | 141 +++++++++++ .../main-quota-provenance.test.ts | 73 ++++++ tests/fixtures/test-layout-expected.json | 2 +- .../direct-credit-renewal-ingress.test.ts | 3 +- 13 files changed, 525 insertions(+), 21 deletions(-) create mode 100644 tests/codex-integration/codex-credit-retry-after.test.ts create mode 100644 tests/codex-integration/codex-credit-scoped-retry.test.ts diff --git a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md index d2e2ee8c2e3..6d96a5d8a98 100644 --- a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md +++ b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md @@ -48,6 +48,25 @@ that the account pool routed to the main account, the earliest known time anothe pool account recovers, whichever comes first (never sooner than one minute). Resume the Goal after that time; routing then picks an account from fresh state. +## Stored-account credit metadata retries + +An explicitly selected stored account can reach its included usage limit while its last +credit balance observation is older than five minutes. If credit spending is enabled for +that account and the evidence is stale, missing or incomplete, its local 429 advises retrying +after the 300-second base metadata cadence (or at the included reset if sooner). A later known upstream +cooldown for the selected quota scope, including account-wide holds, still takes precedence. Unrelated +model-group holds are not imported. The original included-window reset remains attached to the +internal refusal as `resetAt`; the shorter hint is a metadata recheck opportunity, not a new quota reset. + +The request remains refused until fresh spendable credit evidence is available. This hint +does not perform a refresh, bypass metadata-query pacing, guarantee recovery, change account +selection, or enable credit spending. Direct account bindings retain their identity. Explicit +credit opt-out, an older cached null without provenance, a known spending veto or a finite empty +balance without unlimited entitlement retain the included-reset advice. A newly observed missing +or malformed credit summary retracts spending authority but permits the short metadata retry; +an explicit spending veto still wins. Unlimited credits with a zero balance require fresh evidence +just like other credits. Ordinary upstream 429 delays are not capped by this policy. + ## Why the first reconnect row can be absent The public Codex `rust-v0.158.0-alpha.2.1` source gates the ordinary stream-retry diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 1e5bd01048c..e477b4f7f50 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -94,7 +94,7 @@ "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", - "codex-pool-credit-policy.test.ts": "codex-integration", + "codex-pool-credit-policy.test.ts": "codex-integration", "codex-credit-retry-after.test.ts": "codex-integration", "codex-credit-scoped-retry.test.ts": "codex-integration", "codex-sidecar-credit-policy.test.ts": "codex-integration", "codex-credits-after-limit-main.test.ts": "codex-integration", "service-runtime-preflight.test.ts": "service", "service-desktop-runtime-preflight.test.ts": "service", diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index 5afdd2230fd..f8ba620d6b2 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1,4 +1,4 @@ -import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt } from "./pool-credit-policy"; +import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt, poolCreditRecoveryNeeded, poolContextCreditRecoveryNeeded } from "./pool-credit-policy"; import { hasSpendableCodexCredits } from "./quota-types"; import { nextQuotaQueryDelay } from "./quota-query-backoff"; import { noteMainAccountActivity } from "./main-account-external-usage"; @@ -548,17 +548,21 @@ export class CodexMainAccountHardLockError extends CodexAccountCooldownError { export class CodexPoolAccountCreditsOffError extends CodexAccountCooldownError { readonly resetAt: number; - constructor(accountId: string, resetAt: number) { - super(accountId, resetAt); + constructor(accountId: string, resetAt: number, creditRecovery = false, quotaScope?: CodexQuotaScope) { + // Keep the included reset; a metadata hint neither admits nor clears an upstream hold. + super(accountId, creditRecovery ? Math.max(Math.min(resetAt, Date.now() + nextQuotaQueryDelay()), + getCodexQuotaHealthSnapshot(accountId, quotaScope)?.cooldownUntil ?? 0) : resetAt); this.name = "CodexPoolAccountCreditsOffError"; this.resetAt = resetAt; - this.message = "The selected Codex account reached its usage limit, and spending credits is off or no fresh spendable balance is available." + this.message = creditRecovery + ? "The selected Codex account reached its usage limit and needs a fresh credit observation. Retry later; spending remains blocked until fresh spendable credits are observed." + : "The selected Codex account reached its usage limit, and spending credits is off or no fresh spendable balance is available." + " Wait for the limit to reset, choose another account, or explicitly allow credits for this account in Codex Auth."; } } -function assertPoolAccountCredits(accountId: string, resetAt: number | undefined): void { - if (resetAt !== undefined) throw new CodexPoolAccountCreditsOffError(accountId, resetAt); +function assertPoolAccountCredits(accountId: string, resetAt: number | undefined, creditRecovery: boolean, quotaScope?: CodexQuotaScope): void { + if (resetAt !== undefined) throw new CodexPoolAccountCreditsOffError(accountId, resetAt, creditRecovery, quotaScope); } /** The main login may not spend credits and one of its usage windows is full (#6334). */ @@ -780,7 +784,7 @@ export function createCodexAuthDispatchGuard( return headers => { if (config.pausedCodexAccountIds?.includes(ctx.accountId)) throw new CodexPoolAuthenticationError("Selected Codex account is paused; resume it in account settings"); if (ctx.kind === "main-pool") { reserveGuard?.(headers); return; } - assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, config)); + assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, config), poolContextCreditRecoveryNeeded(ctx, config), ctx.quotaScope); reserveGuard?.(headers); }; } @@ -1569,7 +1573,7 @@ async function resolveCodexAuthContextUnannotated( try { assertCodexAccountValidationReady(accountId); if (configuredCodexMainDuplicate(config, accountId)) throw new CodexPoolAuthenticationError(CODEX_MAIN_DUPLICATE_ACTION); - assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config)); + assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config), poolCreditRecoveryNeeded(policy, accountId), quotaScope); } catch (cause) { // Nothing will reach upstream, so give the trial back instead of leaving the held account // unprobeable until the lease deadline lapses (#4701). @@ -1700,7 +1704,7 @@ async function resolveCodexAuthContextUnannotated( const token = await getValidCodexToken(accountId, { signal: options.signal }); assertCodexAccountValidationReady(accountId); if (configuredCodexMainDuplicate(config, accountId)) throw new CodexPoolAuthenticationError(CODEX_MAIN_DUPLICATE_ACTION); - assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config)); + assertPoolAccountCredits(accountId, poolCreditHoldResetAt(policy, accountId, config), poolCreditRecoveryNeeded(policy, accountId), quotaScope); return bindPoolCreditPolicy({ kind: "pool", accountId, @@ -1732,7 +1736,7 @@ async function resolveCodexAuthContextUnannotated( export function assertCodexAuthContextNotCooled(ctx: CodexAuthContext | undefined): void { if (ctx?.kind !== "pool" && ctx?.kind !== "main-pool") return; - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx), poolContextCreditRecoveryNeeded(ctx), ctx.quotaScope); // A context holding the probe lease was deliberately admitted through the cooldown. if (ctx.probeLeaseId) return; const cooldown = getCodexQuotaHealthSnapshot(ctx.accountId, ctx.quotaScope); @@ -1748,7 +1752,7 @@ export function applyCodexAuthContextToProvider( ): OcxRuntimeProviderConfig { if (mode !== "pool" || (ctx.kind !== "pool" && ctx.kind !== "main-pool") || provider.authMode !== "forward") return provider; assertCodexAccountValidationReady(ctx.accountId); - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx), poolContextCreditRecoveryNeeded(ctx), ctx.quotaScope); return { ...provider, _codexAccountOverride: { @@ -1803,7 +1807,7 @@ export function materializeCodexUpstreamAuth( } if (ctx.kind === "pool" || ctx.kind === "main-pool") { assertCodexAccountValidationReady(ctx.accountId); - if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, options.config)); + if (ctx.kind === "pool") assertPoolAccountCredits(ctx.accountId, poolContextCreditHoldResetAt(ctx, options.config), poolContextCreditRecoveryNeeded(ctx, options.config), ctx.quotaScope); selected.set("authorization", `Bearer ${ctx.accessToken}`); selected.set("chatgpt-account-id", ctx.chatgptAccountId); if (ctx.kind === "main-pool") { diff --git a/src/codex/pool-credit-policy.ts b/src/codex/pool-credit-policy.ts index d1a69e1eadd..bec446f7e3f 100644 --- a/src/codex/pool-credit-policy.ts +++ b/src/codex/pool-credit-policy.ts @@ -1,8 +1,9 @@ import type { OcxConfig } from "../types"; -import { codexUsageLimitResetAt, isCodexAccountHeldForCredits } from "./account-credit-use"; +import { codexAccountUsesCreditsAfterLimit, codexUsageLimitResetAt, isCodexAccountHeldForCredits } from "./account-credit-use"; import { MAIN_CODEX_ACCOUNT_ID } from "./main-account"; import { isSelectableCodexPoolAccount } from "./account-id"; import { getAccountQuota } from "./quota"; +import { hasSpendableCodexCredits } from "./quota-types"; export type PoolCreditPolicyConfig = Readonly>; type PoolContext = { kind: "pool"; accountId: string }; @@ -65,3 +66,23 @@ export function poolContextCreditHoldResetAt( const catalog = override?.codexAccounts ? override : bound?.config ?? policy; return poolCreditHoldResetAt(policy, context.accountId, catalog, now); } + +/** Retry advice only: absent/uncertain evidence can be re-observed, never spent. */ +export function poolCreditRecoveryNeeded(policy: PoolCreditPolicyConfig, accountId: string, now = Date.now()): boolean { + if (!codexAccountUsesCreditsAfterLimit(policy, accountId)) return false; + const quota = getAccountQuota(accountId); + const credits = quota?.credits; + // Only parser-proven unavailable null is re-observable. Legacy null and known + // vetoes remain conservative; unlimited overrides balance, never an explicit veto. + if (credits === null) return quota?.creditObservationUnavailable === true; + if (credits?.allowed === false || credits?.overageLimitReached === true + || credits?.hasCredits === false || (credits?.unlimited !== true && typeof credits?.balance === "number" + && Number.isFinite(credits.balance) && credits.balance <= 0)) return false; + return !hasSpendableCodexCredits({ credits }, now); +} + +/** Use the same live policy/override as the context's admission check. */ +export function poolContextCreditRecoveryNeeded(context: PoolContext, override?: PoolCreditPolicyConfig): boolean { + const policy = override ?? policies.get(context)?.policy; + return !!policy && poolCreditRecoveryNeeded(policy, context.accountId); +} diff --git a/src/codex/quota-types.ts b/src/codex/quota-types.ts index d3854172f8f..be1a0bb65c4 100644 --- a/src/codex/quota-types.ts +++ b/src/codex/quota-types.ts @@ -108,6 +108,9 @@ export type StoredAccountQuota = { resetCredits?: number; /** Spendable usage credits; resetCredits are separate, manually redeemed reset tickets. */ credits?: CodexSpendableCredits | null; + /** Only with null: provider evidence was unavailable, not a proven spending veto. + * Affects retry advice only; legacy null remains conservative. */ + creditObservationUnavailable?: true; /** Monthly usage came from an explicitly monthly PRIMARY, not supplementary tertiary, window. */ monthlyIsPrimaryWindow?: boolean; updatedAt: number; diff --git a/src/codex/quota.ts b/src/codex/quota.ts index 0088c4968d6..dc1d575f757 100644 --- a/src/codex/quota.ts +++ b/src/codex/quota.ts @@ -386,8 +386,11 @@ function mergeAccountQuota( const creditsOnly = snapshotHasCredits(quota) && !snapshotHasUsage(quota); // An explicit empty/invalid credits summary retracts the old balance. Partial headers // retain the original observation clock instead of extending its spending authority. - if (quota.credits !== undefined) next.credits = quota.credits; - else if (existing?.credits !== undefined) next.credits = existing.credits; + const creditSource = quota.credits !== undefined ? quota : existing; + if (creditSource?.credits !== undefined) next.credits = creditSource.credits; + if (creditSource?.credits === null && creditSource.creditObservationUnavailable === true) { + next.creditObservationUnavailable = true; + } if (creditsOnly) { if (existing?.weeklyPercent !== undefined) next.weeklyPercent = existing.weeklyPercent; @@ -677,6 +680,8 @@ export function updateAccountQuota( ...(existing?.customWindows !== undefined ? { customWindows: existing.customWindows } : {}), ...(existing?.resetCredits !== undefined ? { resetCredits: existing.resetCredits } : {}), ...(existing?.credits !== undefined ? { credits: existing.credits } : {}), + ...(existing?.credits === null && existing.creditObservationUnavailable === true + ? { creditObservationUnavailable: true as const } : {}), updatedAt: Date.now(), }; assignCarriedShort(quota, existing, quota.updatedAt); @@ -736,7 +741,10 @@ function readMainPolicyQuota(value: unknown): MainPolicyQuota | null { } if (quota.monthlyPercent !== undefined && raw.monthlyIsPrimaryWindow === true) quota.monthlyIsPrimaryWindow = true; // Retain the independent credit timestamp on restart; it still expires at its original deadline. - if (raw.credits === null) quota.credits = null; + if (raw.credits === null) { + quota.credits = null; + if (raw.creditObservationUnavailable === true) quota.creditObservationUnavailable = true; + } else if (raw.credits && typeof raw.credits === "object" && !Array.isArray(raw.credits)) { const stored = raw.credits as Record; if (typeof stored.observedAt === "number" && Number.isFinite(stored.observedAt) && stored.observedAt >= 0) { @@ -967,6 +975,9 @@ export function parseUsageQuota(data: WhamUsageResponse): Omit; +let originalFetch: typeof fetch; +let networkCalls = 0; +const cfg = (): OcxConfig => ({ + port: 10100, defaultProvider: "openai", providers: {}, codexMainAccountHardLock: false, + codexAccounts: [{ id: ID, email: "fixture@example.test", plan: "plus", isMain: false }], + codexAccountNamespaces: { subAlias: ID }, creditCodexAccountIds: [ID], + activeCodexAccountId: ID, autoSwitchThreshold: 0, upstreamFailoverThreshold: 3, +}); +function observe(credits: NonNullable[1]>["credits"], resetAt = RESET) { + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: resetAt, ...(credits === undefined ? {} : { credits }) }); +} +const positive = (age = 360_000) => ({ hasCredits: true, balance: 5, allowed: true, observedAt: NOW - age }); +const resolve = (config = cfg()) => resolveCodexAuthContext(new Headers(), config, "direct", { accountId: ID, modelId: "gpt-5.5" }); +async function refusal(config = cfg()): Promise { + const error = await resolve(config).catch(e => e); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(RESET); + expect(isAccountNeedsReauth(ID)).toBe(false); + return error; +} +function expectRetry(error: CodexAccountCooldownError, seconds: number) { + const response = cooldownErrorResponse(error, Date.now()); + expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe(String(seconds)); +} +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; previousCodexHome = process.env.CODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-credit-retry-")); + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + clock = spyOn(Date, "now").mockReturnValue(NOW); + const ok = { success: true, exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + setIcaclsRunnerForTests(() => ok); setAsyncIcaclsRunnerForTests(async () => ok); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + saveCodexAccountCredential(ID, { accessToken: "fixture-credit-access", refreshToken: "fixture-credit-refresh", + expiresAt: NOW + 3_600_000, chatgptAccountId: "fixture-credit-workspace" }); + originalFetch = globalThis.fetch; networkCalls = 0; + globalThis.fetch = Object.assign((): never => { networkCalls++; throw new Error("Unexpected network request"); }, + { preconnect: originalFetch.preconnect }); +}); +afterEach(async () => { + try { + expect(networkCalls).toBe(0); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + await flushConfigDirHardeningForTests(); + } finally { + globalThis.fetch = originalFetch; clock.mockRestore(); + setIcaclsRunnerForTests(null); setAsyncIcaclsRunnerForTests(null); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); + } +}); +for (const kind of ["stale", "missing", "partial", "future", "invalid-time"] as const) test(`opted-in exact Direct account gets a metadata retry for ${kind} evidence`, async () => { + observe(kind === "stale" ? positive() : kind === "future" ? positive(-1) + : kind === "invalid-time" ? { ...positive(), observedAt: NaN } + : kind === "partial" ? { observedAt: NOW } : undefined); + const before = structuredClone(getAccountQuota(ID)); + const error = await refusal(); expectRetry(error, 300); + expect(getAccountQuota(ID)).toEqual(before); + await expect(resolve()).rejects.toBeInstanceOf(CodexPoolAccountCreditsOffError); +}); +test("opt-out retains the actual six-day reset despite stale evidence", async () => { + observe(positive()); const config = cfg(); config.creditCodexAccountIds = []; + expectRetry(await refusal(config), 518_400); +}); +for (const age of [0, 360_000]) for (const [reason, veto] of [ + ["empty", { balance: 0 }], ["spend control", { allowed: false }], + ["overage", { overageLimitReached: true }], ["no credits", { hasCredits: false }], +] as const) test(`known ${reason} retains reset at credit age ${age}`, async () => { + observe({ ...positive(age), ...veto }); expectRetry(await refusal(), 518_400); +}); +test("explicit retraction retains the included reset", async () => { + observe(null); expectRetry(await refusal(), 518_400); +}); +test("spend-control-only revocation remains a reset-bound refusal", async () => { + observe(positive(0)); setAccountQuotaFromParsed(ID, parseUsageQuota({ spend_control: { reached: true } })); + expect(getAccountQuota(ID)?.credits).toBeNull(); expectRetry(await refusal(), 518_400); +}); +test("the included reset earlier than metadata cadence remains the hint", async () => { + observe(positive(), NOW + 30_000); + const error = await resolve().catch(e => e); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(NOW + 30_000); expectRetry(error, 30); +}); +test("TTL boundary admits at five minutes and refuses just beyond it", async () => { + observe(positive(300_000)); const context = await resolve(); expect(context.kind).toBe("pool"); + clock.mockReturnValue(NOW + 1); + expect(() => materializeCodexUpstreamAuth(new Headers(), context)).toThrow(CodexPoolAccountCreditsOffError); +}); +for (const boundary of ["materialize", "dispatch", "sidecar", "provider", "cooldown"] as const) test(`late ${boundary} refusal retains live consent and metadata hint`, async () => { + observe(positive(0)); const config = cfg(); const context = await resolve(config); + observe(positive()); + const run = () => boundary === "materialize" ? materializeCodexUpstreamAuth(new Headers(), context) + : boundary === "dispatch" ? createCodexAuthDispatchGuard(context, config, "gpt-5.5")!(new Headers()) + : boundary === "provider" ? applyCodexAuthContextToProvider({ adapter: "openai-responses", baseUrl: "https://example.test", authMode: "forward" }, context, "pool") + : boundary === "cooldown" ? assertCodexAuthContextNotCooled(context) + : createOpenAiSidecarCreditGuard(context, config)!(); + let error: unknown; try { run(); } catch (e) { error = e; } + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); expectRetry(error as CodexAccountCooldownError, 300); + config.creditCodexAccountIds = []; + try { run(); } catch (e) { error = e; } + expectRetry(error as CodexAccountCooldownError, 518_400); +}); +test("authoritative upstream 429 Retry-After is not capped", () => { + const error = new CodexAccountCooldownError(ID, NOW + 3_600_000, "retry-after"); + expectRetry(error, 3_600); +}); +test("a stored upstream 429 cooldown still dominates a metadata-only retry hint", async () => { + observe(positive()); + recordCodexUpstreamOutcome(cfg(), ID, 429, { retryAfter: "3600", now: NOW }); + expectRetry(await refusal(), 3_600); +}); + +test("materialization override wins over bound credit consent in both directions", async () => { + observe(positive(0)); const config = cfg(); const context = await resolve(config); observe(positive()); + const attempt = (allowed: boolean) => { + try { materializeCodexUpstreamAuth(new Headers(), context, { config: { creditCodexAccountIds: allowed ? [ID] : [] } }); } + catch (error) { expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); return error as CodexPoolAccountCreditsOffError; } + throw new Error("expected credit admission refusal"); + }; + expectRetry(attempt(false), 518_400); + config.creditCodexAccountIds = []; + const error = attempt(true); expect(error.resetAt).toBe(RESET); expectRetry(error, 300); +}); + +test("unlimited zero balance admits while fresh and requests metadata after expiry", async () => { + observe({ ...positive(300_000), unlimited: true, balance: 0 }); + expect((await resolve()).kind).toBe("pool"); + clock.mockReturnValue(NOW + 1); + expectRetry(await refusal(), 300); +}); +for (const veto of [{ allowed: false }, { hasCredits: false }, { overageLimitReached: true }]) { + test(`unlimited zero balance preserves explicit veto ${JSON.stringify(veto)}`, async () => { + observe({ ...positive(), unlimited: true, balance: 0, ...veto }); + expectRetry(await refusal(), 518_400); + }); +} +const unavailablePayloads: Parameters[0][] = [ + { credits: {} }, { credits: null }, { credits: { balance: "invalid" } }, + { rate_limit: { allowed: false } }, + { credits: "invalid" } as unknown as Parameters[0], +]; +for (const payload of unavailablePayloads) test(`parser uncertainty stays refused with a short retry: ${JSON.stringify(payload)}`, async () => { + observe(positive(0)); + setAccountQuotaFromParsed(ID, parseUsageQuota(payload)); + expect(getAccountQuota(ID)?.credits).toBeNull(); + expectRetry(await refusal(), 300); + updateAccountQuota(ID, 100, RESET); + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: RESET }); + expectRetry(await refusal(), 300); + const config = cfg(); config.creditCodexAccountIds = []; + expectRetry(await refusal(config), 518_400); +}); +for (const payload of unavailablePayloads) for (const spend_control of [{ reached: true }, {}]) { + test(`spending veto dominates unavailable evidence: ${JSON.stringify({ ...payload, spend_control })}`, async () => { + observe(positive(0)); + setAccountQuotaFromParsed(ID, parseUsageQuota({ ...payload, spend_control })); + expectRetry(await refusal(), 518_400); + }); +} +test("replacement clears uncertainty provenance, without reviving credit authority", async () => { + observe(positive(0)); + const context = await resolve(); + const unknown = () => setAccountQuotaFromParsed(ID, parseUsageQuota({ credits: {} })); + unknown(); expectRetry(await refusal(), 300); + expect(() => materializeCodexUpstreamAuth(new Headers(), context)).toThrow(CodexPoolAccountCreditsOffError); + observe(null); expectRetry(await refusal(), 518_400); + unknown(); observe({ ...positive(), allowed: false }); expectRetry(await refusal(), 518_400); + unknown(); observe(positive(0)); expect((await resolve()).kind).toBe("pool"); + expect(getAccountQuota(ID)?.creditObservationUnavailable).toBeUndefined(); + unknown(); observe(positive()); expectRetry(await refusal(), 300); + expect(getAccountQuota(ID)?.creditObservationUnavailable).toBeUndefined(); + unknown(); setAccountQuotaFromParsed(ID, parseUsageQuota({ spend_control: { reached: true } })); + expectRetry(await refusal(), 518_400); + unknown(); expectRetry(await refusal(), 300); +}); +test("unavailable versus legacy null survives persistence and a fresh reader process", async () => { + observe(positive()); + setAccountQuotaFromParsed(ID, parseUsageQuota({ credits: {} })); + setAccountQuotaFromParsed("legacy-null", { credits: null }); + const path = join(home, "codex-quota-cache.json"); + for (let tries = 0; !existsSync(path) && tries < 100; tries++) await Bun.sleep(10); + expect(existsSync(path)).toBe(true); + const child = Bun.spawnSync([process.execPath, "-e", ` + import { getAccountQuota } from './src/codex/quota'; + import { poolCreditRecoveryNeeded } from './src/codex/pool-credit-policy'; + Date.now = () => ${NOW}; + const policy = {creditCodexAccountIds: ['${ID}', 'legacy-null']}; + console.log(JSON.stringify(['${ID}', 'legacy-null'].map(id => ({ + credits: getAccountQuota(id)?.credits, recovery: poolCreditRecoveryNeeded(policy, id) + })))); + `], { cwd: repoRoot(), env: process.env, stdout: "pipe", stderr: "pipe" }); + expect(child.exitCode).toBe(0); + expect(JSON.parse(child.stdout.toString())).toEqual([ + { credits: null, recovery: true }, { credits: null, recovery: false }, + ]); +}); diff --git a/tests/codex-integration/codex-credit-scoped-retry.test.ts b/tests/codex-integration/codex-credit-scoped-retry.test.ts new file mode 100644 index 00000000000..1946f3e3005 --- /dev/null +++ b/tests/codex-integration/codex-credit-scoped-retry.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OcxConfig } from "../../src/types"; +import { + CodexPoolAccountCreditsOffError, cooldownErrorResponse, resolveCodexAuthContext, + materializeCodexUpstreamAuth, materializeCodexUpstreamAuthAsync, + createCodexAuthDispatchGuard, applyCodexAuthContextToProvider, assertCodexAuthContextNotCooled, +} from "../../src/codex/auth-context"; +import { createOpenAiSidecarCreditGuard } from "../../src/providers/openai-sidecar-credit"; +import { saveCodexAccountCredential } from "../../src/codex/account-store"; +import { clearAccountNeedsReauth, isAccountNeedsReauth } from "../../src/codex/account-runtime-state"; +import { clearAccountQuota, getAccountQuota, setAccountQuotaFromParsed } from "../../src/codex/quota"; +import { + clearCodexUpstreamHealth, clearThreadAccountMap, getCodexQuotaHealthSnapshot, recordCodexUpstreamOutcome, +} from "../../src/codex/routing"; +import { clearPoolRotationState } from "../../src/codex/pool-rotation"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; +import { setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests } from "../../src/lib/windows-secret-acl"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const ID = "scoped-credit-fixture"; +const NOW = 1_800_000_000_000; +const RESET = NOW + 6 * 86_400_000; +const MODEL = "gpt-5.5"; +let home: string; +let previousHome: string | undefined, previousCodexHome: string | undefined; +let clock: ReturnType; +let originalFetch: typeof fetch; +let networkCalls: number; +let expectedNetworkCalls: number; +const cfg = (): OcxConfig => ({ + port: 10100, defaultProvider: "openai", providers: {}, codexMainAccountHardLock: false, + codexAccounts: [{ id: ID, email: "fixture@example.test", plan: "plus", isMain: false }], + codexAccountNamespaces: { subAlias: ID }, creditCodexAccountIds: [ID], + activeCodexAccountId: ID, autoSwitchThreshold: 0, upstreamFailoverThreshold: 3, +}); +function observe(age: number) { + setAccountQuotaFromParsed(ID, { weeklyPercent: 100, weeklyResetAt: RESET, + credits: { hasCredits: true, balance: 5, allowed: true, observedAt: NOW - age } }); +} +const resolve = (config: OcxConfig) => resolveCodexAuthContext(new Headers(), config, "direct", { accountId: ID, modelId: MODEL }); +function state() { + return structuredClone({ quota: getAccountQuota(ID), + account: getCodexQuotaHealthSnapshot(ID, undefined), + shared: getCodexQuotaHealthSnapshot(ID, "shared"), + reserve: getCodexQuotaHealthSnapshot(ID, "reserve") }); +} +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; previousCodexHome = process.env.CODEX_HOME; + home = mkdtempSync(join(tmpdir(), "ocx-credit-scoped-retry-")); + process.env.OPENCODEX_HOME = home; process.env.CODEX_HOME = home; + clock = spyOn(Date, "now").mockReturnValue(NOW); + const ok = { success: true, exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + setIcaclsRunnerForTests(() => ok); setAsyncIcaclsRunnerForTests(async () => ok); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + saveCodexAccountCredential(ID, { accessToken: "fixture-scoped-access", refreshToken: "fixture-scoped-refresh", + expiresAt: NOW + 3_600_000, chatgptAccountId: "fixture-scoped-workspace" }); + originalFetch = globalThis.fetch; networkCalls = 0; expectedNetworkCalls = 0; + globalThis.fetch = Object.assign((): never => { networkCalls++; throw new Error("Unexpected network request"); }, + { preconnect: originalFetch.preconnect }); +}); +afterEach(async () => { + try { + expect(networkCalls).toBe(expectedNetworkCalls); + clearAccountQuota(); clearCodexUpstreamHealth(); clearThreadAccountMap(); clearPoolRotationState(); clearAccountNeedsReauth(ID); + await flushConfigDirHardeningForTests(); + } finally { + globalThis.fetch = originalFetch; clock.mockRestore(); + setIcaclsRunnerForTests(null); setAsyncIcaclsRunnerForTests(null); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); + } +}); + +const cases = [ + { name: "shared reset-derived hold", modelId: MODEL, retry: 900 }, + { name: "independent Reserve hold", modelId: "gpt-reserve", retry: 300 }, + { name: "account-wide Retry-After", modelId: "gpt-reserve", retryAfter: "3600", retry: 3600 }, + { name: "expired shared hold", modelId: MODEL, age: 901_000, retry: 300 }, +] as const; +for (const boundary of ["resolve", "materialize", "materialize-async", "dispatch", "provider", "cooldown", "sidecar"] as const) { + for (const scenario of cases) test(`${boundary} credit refusal respects ${scenario.name}`, async () => { + const config = cfg(); + observe(0); + const context = boundary === "resolve" ? undefined : await resolve(config); + if (context) expect(context).toMatchObject({ kind: "pool", accountId: ID, quotaScope: "shared" }); + observe(360_000); + const recordedAt = NOW - ("age" in scenario ? scenario.age : 0); + recordCodexUpstreamOutcome(config, ID, 429, { modelId: scenario.modelId, + resetAt: recordedAt + 3_600_000, fixedAccount: true, now: recordedAt, + ...("retryAfter" in scenario ? { retryAfter: scenario.retryAfter } : {}) }); + const before = state(); + let error: unknown; + try { + if (boundary === "resolve") await resolve(config); + else if (boundary === "materialize") materializeCodexUpstreamAuth(new Headers(), context!); + else if (boundary === "materialize-async") await materializeCodexUpstreamAuthAsync(new Headers(), context!); + else if (boundary === "dispatch") createCodexAuthDispatchGuard(context!, config, MODEL)!(new Headers()); + else if (boundary === "provider") applyCodexAuthContextToProvider({ + adapter: "openai-responses", baseUrl: "https://example.test", authMode: "forward", + }, context!, "pool"); + else if (boundary === "cooldown") assertCodexAuthContextNotCooled(context!); + else createOpenAiSidecarCreditGuard(context!, config)!(); + } catch (cause) { error = cause; } + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + const refusal = error as CodexPoolAccountCreditsOffError; + expect(refusal.accountId).toBe(ID); + expect(refusal.resetAt).toBe(RESET); + const response = cooldownErrorResponse(refusal, NOW); + expect(response.status).toBe(429); + expect(response.headers.get("retry-after")).toBe(String(scenario.retry)); + expect(state()).toEqual(before); + expect(isAccountNeedsReauth(ID)).toBe(false); + }); +} + +test("resolve retains shared scope when credits become stale during token preparation", async () => { + const config = cfg(); + observe(0); + saveCodexAccountCredential(ID, { accessToken: "fixture-expired-access", refreshToken: "fixture-scoped-refresh", + expiresAt: NOW - 1, chatgptAccountId: "fixture-scoped-workspace" }); + expectedNetworkCalls = 1; + globalThis.fetch = Object.assign(async (input: Parameters[0], init?: RequestInit) => { + networkCalls++; + expect(String(input)).toBe("https://auth.openai.com/oauth/token"); + expect(init?.method).toBe("POST"); + observe(360_000); + recordCodexUpstreamOutcome(config, ID, 429, { modelId: MODEL, + resetAt: NOW + 3_600_000, fixedAccount: true, now: NOW }); + return Response.json({ access_token: "fixture-refreshed-access", refresh_token: "fixture-refreshed-grant", expires_in: 3600 }); + }, { preconnect: originalFetch.preconnect }) as typeof fetch; + const error = await resolve(config).catch(cause => cause); + expect(error).toBeInstanceOf(CodexPoolAccountCreditsOffError); + expect(error.resetAt).toBe(RESET); + expect(cooldownErrorResponse(error, NOW).headers.get("retry-after")).toBe("900"); + expect(getCodexQuotaHealthSnapshot(ID, "shared")?.cooldownUntil).toBe(NOW + 900_000); + expect(isAccountNeedsReauth(ID)).toBe(false); +}); diff --git a/tests/codex-integration/main-quota-provenance.test.ts b/tests/codex-integration/main-quota-provenance.test.ts index d8cb4b4fd65..bb227bcd716 100644 --- a/tests/codex-integration/main-quota-provenance.test.ts +++ b/tests/codex-integration/main-quota-provenance.test.ts @@ -5,6 +5,8 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { MAIN_CODEX_ACCOUNT_ID as MAIN } from "../../src/codex/account-id"; +import { isCodexAccountHeldForCredits } from "../../src/codex/account-credit-use"; +import { hasSpendableCodexCredits } from "../../src/codex/quota-types"; import { getMainAccountHardLockStatus } from "../../src/codex/main-account-hard-lock"; import { applyConfirmedMainCodexAccountTransition, @@ -341,6 +343,77 @@ describe("main policy quota durability and lifecycle", () => { }); }, COLD_SPAWN_WARMUP_HOOK_BUDGET_MS); + for (const unavailable of [true, false]) { + test(`persisted ${unavailable ? "unavailable" : "legacy"} null remains identity-bound and cannot authorize spending`, () => { + const writer = writerFor(); + const resetAt = Date.now() + 86_400_000; + setAccountQuotaFromParsed(MAIN, { weeklyPercent: 100, weeklyResetAt: resetAt, + credits: { hasCredits: true, balance: 5, observedAt: Date.now() } }, undefined, writer); + setAccountQuotaFromParsed(MAIN, unavailable ? parseUsageQuota({ credits: {} }) : { credits: null }, + undefined, writer); + // Partial headers carry the pair without supplying new spending evidence. + applyAccountQuotaFromUpstreamHeaders(MAIN, new Headers({ "x-codex-primary-used-percent": "100", + "x-codex-primary-reset-at": String(resetAt) }), + undefined, writer); + const quota = getMainPolicyQuota(); + expect(quota?.credits).toBeNull(); + expect(quota?.creditObservationUnavailable).toBe(unavailable ? true : undefined); + expect(hasSpendableCodexCredits(quota)).toBe(false); + expect(isCodexAccountHeldForCredits({ creditCodexAccountIds: [MAIN] }, MAIN, quota, "plus", Date.now())).toBe(true); + const persisted = JSON.parse(flushPersistence()); + expect(persisted.mainPolicyQuota).toEqual({ identityKey: writer.identityKey, quota }); + // Read the actual serializer output in a fresh process, without clearing or rewriting it. + const script = ` + ${QUOTA_PROVENANCE_IMPORT_PROLOGUE} + import { hasSpendableCodexCredits } from ${JSON.stringify(repoPath("src/codex/quota-types.ts"))}; + import { isCodexAccountHeldForCredits } from ${JSON.stringify(repoPath("src/codex/account-credit-use.ts"))}; + globalThis.fetch = () => { throw new Error("Unexpected network request"); }; + const before = getMainPolicyQuota(); + observeMainQuotaIdentity("fixture-main-b"); + const other = getMainPolicyQuota(); + observeMainQuotaIdentity("fixture-main-a"); + const policy = getMainPolicyQuota(); + console.log(JSON.stringify({ before, other, policy, + spendable: hasSpendableCodexCredits(policy), + held: isCodexAccountHeldForCredits({ creditCodexAccountIds: ["__main__"] }, "__main__", policy, "plus", Date.now()) })); + `; + const child = Bun.spawnSync({ + cmd: [process.execPath, "--eval", script], cwd: repoRoot(), env: process.env, + timeout: process.platform === "win32" ? SPAWN_BUDGET_MS - INTERNAL_DEADLINE_MS : 10_000, + }); + expect(child.exitCode).toBe(0); + expect(JSON.parse(child.stdout.toString())).toEqual({ before: null, other: null, policy: quota, + spendable: false, held: true }); + }, SPAWN_BUDGET_MS); + } + + test("stale main writers cannot replace unavailable evidence or attach it to a new identity", () => { + const oldA = writerFor(); + const unavailable = parseUsageQuota({ credits: {} }); + const positive = { hasCredits: true, balance: 5, observedAt: Date.now() }; + setAccountQuotaFromParsed(MAIN, unavailable, undefined, oldA); + const writerB = writerFor("fixture-main-b"); + expect(getMainPolicyQuota()).toBeNull(); + setAccountQuotaFromParsed(MAIN, { credits: positive }, undefined, writerB); + setAccountQuotaFromParsed(MAIN, unavailable, undefined, oldA); + expect(getMainPolicyQuota()?.credits).toEqual(positive); + expect(getMainPolicyQuota()?.creditObservationUnavailable).toBeUndefined(); + + const newA = writerFor(); + setAccountQuotaFromParsed(MAIN, { weeklyPercent: 100, weeklyResetAt: Date.now() + 86_400_000 }, undefined, newA); + setAccountQuotaFromParsed(MAIN, unavailable, undefined, newA); + const before = getMainPolicyQuota(); + expect(before).toMatchObject({ credits: null, creditObservationUnavailable: true }); + // Returning to A does not revive a writer captured before the A → B → A transition. + setAccountQuotaFromParsed(MAIN, { credits: positive }, undefined, oldA); + expect(isMainQuotaWriterLive(oldA)).toBe(false); + expect(getMainPolicyQuota()).toEqual(before); + expect(getAccountQuota(MAIN)?.credits).toBeNull(); + expect(hasSpendableCodexCredits(getMainPolicyQuota())).toBe(false); + expect(isCodexAccountHeldForCredits({ creditCodexAccountIds: [MAIN] }, MAIN, getMainPolicyQuota(), "plus", Date.now())).toBe(true); + expect(JSON.parse(flushPersistence()).mainPolicyQuota).toEqual({ identityKey: newA.identityKey, quota: before }); + }); + for (const resetAt of [undefined, 4_000_000_000]) { test(`restart beyond six hours retains ${resetAt ? "future-reset" : "missing-reset"} policy evidence only for observed A`, () => { const writer = writerFor(); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index df9c4ef9ced..96c898215f5 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -170,7 +170,7 @@ "codex-config-preservation.test.ts": "codex-integration", "codex-credits.test.ts": "codex-integration", "codex-credits-settings.test.ts": "codex-integration", "codex-credits-probes.test.ts": "codex-integration", "codex-credits-after-limit.test.ts": "codex-integration", - "codex-pool-credit-policy.test.ts": "codex-integration", + "codex-pool-credit-policy.test.ts": "codex-integration", "codex-credit-retry-after.test.ts": "codex-integration", "codex-credit-scoped-retry.test.ts": "codex-integration", "codex-direct-credit-renewal.test.ts": "codex-integration", "codex-direct-renewal-revocation.test.ts": "codex-integration", "direct-credit-renewal-ingress.test.ts": "responses", "codex-sidecar-credit-policy.test.ts": "codex-integration", diff --git a/tests/responses/direct-credit-renewal-ingress.test.ts b/tests/responses/direct-credit-renewal-ingress.test.ts index c722ca9367b..3df9a1c9fe2 100644 --- a/tests/responses/direct-credit-renewal-ingress.test.ts +++ b/tests/responses/direct-credit-renewal-ingress.test.ts @@ -112,7 +112,7 @@ async function send() { body: JSON.stringify({ model: "subAlias/gpt-5.5", input: "fixture", stream: false, store: false }), }); const response = await handleResponses(request, config, { model: "", provider: "" }); - return { status: response.status, body: await response.json() }; + return { status: response.status, retryAfter: response.headers.get("retry-after"), body: await response.json() }; } test("same Direct namespace recovers after five-minute credit expiry without fallback or external refresh", async () => { @@ -123,6 +123,7 @@ test("same Direct namespace recovers after five-minute credit expiry without fal now += 360_001; const stale = await send(); expect(stale.status).toBe(429); + expect(stale.retryAfter).toBe("300"); expect(responseAccounts).toHaveLength(1); expect(metadataAccounts).toEqual([]); const noMain = () => { throw new Error("stored-account renewal must never access main"); }; From 268adf6a9dd660814025039e0ae9fbd29d0a653d Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:56:58 +0900 Subject: [PATCH 3/6] fix(test): supply required port in native pool fixture --- tests/helpers/compaction-routing-fixtures.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/helpers/compaction-routing-fixtures.ts b/tests/helpers/compaction-routing-fixtures.ts index 3ae7cef893a..616b9350dc2 100644 --- a/tests/helpers/compaction-routing-fixtures.ts +++ b/tests/helpers/compaction-routing-fixtures.ts @@ -61,6 +61,7 @@ export function keyProviderConfig(overrides: Partial = {}): O export function nativePoolConfig(): OcxConfig { return { + port: 0, defaultProvider: "openai", activeCodexAccountId: "pool-a", providers: { From fdec8897f168255efed2321490d02bb5f5345614 Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:15:21 +0900 Subject: [PATCH 4/6] test(codex): isolate ACL setup and cold quota child work --- tests/responses/ws-upstream.test.ts | 2 ++ tests/usage/quota-pool.test.ts | 18 ++++++++++-------- tests/usage/quota-reset-observation.test.ts | 8 +++++++- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/responses/ws-upstream.test.ts b/tests/responses/ws-upstream.test.ts index 9d587853a5f..b930612be0a 100644 --- a/tests/responses/ws-upstream.test.ts +++ b/tests/responses/ws-upstream.test.ts @@ -33,6 +33,8 @@ import type { OcxProviderConfig } from "../../src/types"; import type { OcxConfig } from "../../src/types"; import { BOUNDED_WS_RUNTIME, codexWsUpstreamFetch, shouldUseCodexWsUpstream, streamingInit } from "../helpers/ws-upstream-fixtures"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { installCompactionRoutingAclFixture as installRoutingAclFixture } from "../helpers/compaction-routing-fixtures"; +installRoutingAclFixture(); const CODEX_URL = "https://chatgpt.com/backend-api/codex/responses"; diff --git a/tests/usage/quota-pool.test.ts b/tests/usage/quota-pool.test.ts index 29a13b02baf..6f613e7f271 100644 --- a/tests/usage/quota-pool.test.ts +++ b/tests/usage/quota-pool.test.ts @@ -3,7 +3,7 @@ * a local mirror of the contract, absent-not-zero, the window-kind mapping through the existing * Codex header parser, and the opaque, stable `accountRef`. */ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import { lstatSync, mkdtempSync, readdirSync, readFileSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -22,6 +22,7 @@ import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from ". import { setProviderQuotaReportCache } from "../../src/providers/quota/report-cache"; import { nullPaths, QuotaPoolSchema } from "../helpers/quota-pool-schema"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { COLD_SPAWN_WARMUP_HOOK_BUDGET_MS, warmModuleGraph } from "../helpers/cold-spawn-warmup"; const NOW = Date.parse("2026-10-21T12:00:00.000Z"); const HOUR = 3_600_000; @@ -30,6 +31,12 @@ let home = ""; let priorHome: string | undefined; let admissionPath = ""; let xaiIds: string[] = []; +const IDENTITY_RACE_SCRIPT = `const { ensureQuotaPoolIdentity } = await import(${JSON.stringify(join(import.meta.dir, "../../src/quota-pool/identity.ts"))}); +await Bun.sleep(Math.max(0, Number(process.env.QP_START_AT) - Date.now())); +console.log(ensureQuotaPoolIdentity().homeId);`; +beforeAll(async () => { + await warmModuleGraph({ graph: "quota-pool/identity-race", source: IDENTITY_RACE_SCRIPT }); +}, COLD_SPAWN_WARMUP_HOOK_BUDGET_MS); function config(): OcxConfig { return { @@ -234,14 +241,10 @@ describe("quota-pool identity creation", () => { expect(readQuotaPoolIdentity()).toEqual(identity); }); - test("servers racing on a fresh home agree on one identity", async () => { - const script = `const { ensureQuotaPoolIdentity } = await import(${JSON.stringify(join(import.meta.dir, "../../src/quota-pool/identity.ts"))}); -await Bun.sleep(Math.max(0, Number(process.env.QP_START_AT) - Date.now())); -console.log(ensureQuotaPoolIdentity().homeId);`; - for (let round = 0; round < 3; round++) { + test.each([0, 1, 2])("servers racing on a fresh home agree on one identity (round %i)", async () => { const dir = mkdtempSync(join(home, "race-")); const startAt = String(Date.now() + 400); - const runs = Array.from({ length: 4 }, () => Bun.spawn([process.execPath, "-e", script], { + const runs = Array.from({ length: 4 }, () => Bun.spawn([process.execPath, "-e", IDENTITY_RACE_SCRIPT], { env: { ...process.env, OPENCODEX_HOME: dir, QP_START_AT: startAt }, stdout: "pipe", stderr: "pipe", })); // Read each child to completion (stdout, stderr and exit) so a loser that fails names its error. @@ -252,7 +255,6 @@ console.log(ensureQuotaPoolIdentity().homeId);`; resetQuotaPoolIdentityMemoForTests(); const persisted = readQuotaPoolIdentity(dir)!.homeId; expect(results).toEqual(Array.from({ length: 4 }, () => ({ id: persisted, code: 0, error: "" }))); - } }); }); diff --git a/tests/usage/quota-reset-observation.test.ts b/tests/usage/quota-reset-observation.test.ts index fdf00708ea0..28cbc7a21ae 100644 --- a/tests/usage/quota-reset-observation.test.ts +++ b/tests/usage/quota-reset-observation.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { afterEach, beforeAll, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -23,6 +23,7 @@ import { stopQuotaResetPoller, } from "../../src/quota/reset-poller"; import { resetQuotaResetNotifyCacheForTests } from "../../src/quota/reset-notify-config"; +import { COLD_SPAWN_WARMUP_HOOK_BUDGET_MS, warmModuleGraph } from "../helpers/cold-spawn-warmup"; const ACCOUNT = "acct_reset_observation"; const HOUR = 60 * 60_000; @@ -325,6 +326,11 @@ describe("idle poller", () => { }); describe("observation ordering under a burst", () => { + beforeAll(async () => { + // Compile in a separate process; the assertion still starts with a fresh module registry. + await warmModuleGraph({ graph: "quota-reset/cold-burst", entry: + fileURLToPath(new URL("../helpers/quota-reset-burst-child.ts", import.meta.url)) }); + }, COLD_SPAWN_WARMUP_HOOK_BUDGET_MS); test("a rising-usage burst in a COLD process fires nothing", async () => { // The regression this pins: the seam awaited two import() calls before swapping the // baseline, and Bun does not resolve concurrent dynamic imports in call order. A burst of From 598ccbf96daeca973e267428e2800637f39293b4 Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:20:42 +0900 Subject: [PATCH 5/6] test(codex): complete fetch and routing fixture types --- tests/responses/ws-upstream.test.ts | 64 ++++++++++++++--------------- tests/usage/quota-pool.test.ts | 1 + 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/tests/responses/ws-upstream.test.ts b/tests/responses/ws-upstream.test.ts index b930612be0a..1522e4bb192 100644 --- a/tests/responses/ws-upstream.test.ts +++ b/tests/responses/ws-upstream.test.ts @@ -84,7 +84,7 @@ describe("shouldUseCodexWsUpstream", () => { const response = await rawCodexWsUpstreamFetch( CODEX_URL, streamingInit(), - (async () => sentinel) as typeof fetch, + Object.assign(async () => sentinel, { preconnect: RealFetch.preconnect }), "1.3.14", ); expect(response).toBe(sentinel); @@ -94,7 +94,7 @@ describe("shouldUseCodexWsUpstream", () => { const canaryResponse = await rawCodexWsUpstreamFetch( CODEX_URL, streamingInit(), - (async () => canarySentinel) as typeof fetch, + Object.assign(async () => canarySentinel, { preconnect: RealFetch.preconnect }), { version: "1.4.0", versionWithSha: "v1.4.0-canary.1 (0123abcd)" }, ); expect(canaryResponse).toBe(canarySentinel); @@ -244,11 +244,11 @@ describe("providerFetch routing", () => { test("a canary runtime identity cannot open the WS transport", async () => { const sentinel = new Response("base"); let baseCalls = 0; - const provider = { - fetch: (async () => { + const provider = { adapter: "openai-responses", baseUrl: CODEX_URL, + fetch: Object.assign(async () => { baseCalls += 1; return sentinel; - }) as typeof fetch, + }, { preconnect: RealFetch.preconnect }), } as OcxProviderConfig; const wrapped = providerFetch(provider, { version: "1.4.0", @@ -566,13 +566,13 @@ describe("handleResponses Codex WS relay selection", () => { test("an HTTP fallback remains on the configured legacy tee path", async () => { installFake(ws => ws.close()); - globalThis.fetch = (async () => new Response( + globalThis.fetch = Object.assign(async () => new Response( `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { id: "r-http", status: "completed", output: [] }, })}\n\n`, { status: 200, headers: { "content-type": "text/event-stream" } }, - )) as typeof fetch; + ), { preconnect: RealFetch.preconnect }); takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }, { @@ -595,7 +595,7 @@ describe("handleResponses Codex WS relay selection", () => { } }); - const logCtx = { model: "", provider: "" }; + const logCtx: Parameters[2] = { model: "", provider: "" }; takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, @@ -615,7 +615,7 @@ describe("handleResponses Codex WS relay selection", () => { ws.emit("close", { code: 1006 }); }); - const logCtx = { model: "", provider: "" }; + const logCtx: Parameters[2] = { model: "", provider: "" }; takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), logCtx, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME, @@ -633,13 +633,13 @@ describe("handleResponses Codex WS relay selection", () => { test.skipIf(bunSupportsBoundedCodexWsRelay())( "an older runtime stays on HTTP SSE without opening a WebSocket", async () => { - globalThis.fetch = (async () => new Response( + globalThis.fetch = Object.assign(async () => new Response( `event: response.completed\ndata: ${JSON.stringify({ type: "response.completed", response: { id: "r-old", status: "completed", output: [] }, })}\n\n`, { status: 200, headers: { "content-type": "text/event-stream" } }, - )) as typeof fetch; + ), { preconnect: RealFetch.preconnect }); takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); @@ -674,10 +674,10 @@ describe("handleResponses Codex WS relay selection", () => { }], }, }; - globalThis.fetch = (async () => new Response( + globalThis.fetch = Object.assign(async () => new Response( `event: response.completed\ndata: ${JSON.stringify(upstreamEvent)}\n\n`, { status: 200, headers: { "content-type": "text/event-stream" } }, - )) as typeof fetch; + ), { preconnect: RealFetch.preconnect }); takeSpendHome(); const response = await handleResponses(request(), forwardConfig(), { model: "", provider: "" }); @@ -734,7 +734,7 @@ describe("codexWsUpstreamFetch", () => { headers: { authorization: "Bearer fixture", "content-type": "application/json", "x-openai-internal-codex-responses-lite": "true" }, body: JSON.stringify({ model: "gpt-5.5", input: "hello", stream: true, service_tier: "priority" }), }), { - defaultProvider: "openai", providers: { openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" } }, + port: 0, defaultProvider: "openai", providers: { openai: { adapter: "openai-responses", authMode: "forward", codexAccountMode: "direct", baseUrl: "https://chatgpt.com/backend-api/codex" } }, } as OcxConfig, { model: "", provider: "" }, { codexWsRuntimeIdentity: BOUNDED_WS_RUNTIME }); await response.text(); expect(frames).toHaveLength(1); @@ -797,10 +797,10 @@ describe("codexWsUpstreamFetch", () => { process.env.HTTPS_PROXY = proxy; const sentinel = new Response("sse-fallback"); let fallbackCalls = 0; - const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), (async () => { + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), Object.assign(async () => { fallbackCalls += 1; return sentinel; - }) as typeof fetch); + }, { preconnect: RealFetch.preconnect })); expect(response).toBe(sentinel); expect(fallbackCalls).toBe(1); @@ -886,10 +886,10 @@ describe("codexWsUpstreamFetch", () => { let fallbacks = 0; const response = await fetchWithTransientRetry(() => { attempts++; - return rawCodexWsUpstreamFetch(url, streamingInit(), (async () => { + return rawCodexWsUpstreamFetch(url, streamingInit(), Object.assign(async () => { fallbacks++; throw new Error("a sent create must not be resent over HTTP"); - }) as typeof fetch, BOUNDED_WS_RUNTIME, onQuota); + }, { preconnect: RealFetch.preconnect }), BOUNDED_WS_RUNTIME, onQuota); }, {}); const ws = FakeWebSocket.instances.at(-1)!; expect(attempts).toBe(1); @@ -1091,7 +1091,7 @@ describe("codexWsUpstreamFetch", () => { ws.emit("open", {}); }); const response = await rawCodexWsUpstreamFetch(url, { ...streamingInit(), signal: abort.signal }, - (async () => { fallbacks++; throw new Error("unexpected fallback"); }) as typeof fetch, BOUNDED_WS_RUNTIME); + Object.assign(async () => { fallbacks++; throw new Error("unexpected fallback"); }, { preconnect: RealFetch.preconnect }), BOUNDED_WS_RUNTIME); const ws = FakeWebSocket.instances.at(-1)!; abort.abort(new Error("late abort")); ws.emit("error", {}); @@ -1118,10 +1118,10 @@ describe("codexWsUpstreamFetch", () => { const prepared = prepareCodexWsRequest(CODEX_URL, init)!; const session = new CodexWsSession("wss://chatgpt.com/backend-api/codex/responses", prepared.headers, true); let fallbacks = 0; - const options = { session, url: CODEX_URL, init, prepared, sseFallback: (async () => { + const options = { session, url: CODEX_URL, init, prepared, sseFallback: Object.assign(async () => { fallbacks++; throw new Error("retained create must not fall back"); - }) as typeof fetch }; + }, { preconnect: RealFetch.preconnect }) }; try { expect(session.reserve()).toBe(true); await (await codexWsExchange(options)).text(); @@ -1165,9 +1165,9 @@ describe("codexWsUpstreamFetch", () => { ws.emit("open", {}); ws.emit("message", { data: JSON.stringify(payload, null, 2) }); }); - const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), (async () => { + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), Object.assign(async () => { throw new Error("a sent multiline response cannot fall back"); - }) as typeof fetch); + }, { preconnect: RealFetch.preconnect })); const text = await response.text(); const data = text.split("\n").filter(line => line.startsWith("data: ")); expect(data).toHaveLength(1); @@ -1465,10 +1465,10 @@ describe("codexWsUpstreamFetch", () => { ws.emit("message", { data: JSON.stringify({ type: "codex.response.metadata", headers: { "x-models-etag": "x".repeat(CODEX_WS_METADATA_MAX_BYTES) } }) }); }); let resends = 0; - const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), (async () => { + const response = await codexWsUpstreamFetch(CODEX_URL, streamingInit(), Object.assign(async () => { resends++; return new Response("unexpected resend"); - }) as typeof fetch); + }, { preconnect: RealFetch.preconnect })); expect(response.status).toBe(502); expect(isCodexWsUpstreamResponse(response)).toBe(false); expect(isNonReplayableResponse(response)).toBe(true); @@ -1483,10 +1483,10 @@ describe("codexWsUpstreamFetch", () => { installFake(ws => { ws.emit("open", {}); opened.resolve(); }); let http = 0; try { - const pending = codexWsUpstreamFetch(CODEX_URL, streamingInit(), (async () => { + const pending = codexWsUpstreamFetch(CODEX_URL, streamingInit(), Object.assign(async () => { http++; return new Response("must not resend"); - }) as typeof fetch); + }, { preconnect: RealFetch.preconnect })); await opened.promise; const ws = FakeWebSocket.instances[0]; jest.advanceTimersByTime(28_000); @@ -1514,10 +1514,10 @@ describe("codexWsUpstreamFetch", () => { try { const pending = fetchWithTransientRetry(() => { sends++; - return codexWsUpstreamFetch(CODEX_URL, streamingInit(), (async () => { + return codexWsUpstreamFetch(CODEX_URL, streamingInit(), Object.assign(async () => { http++; return new Response("must not resend"); - }) as typeof fetch); + }, { preconnect: RealFetch.preconnect })); }, {}); await opened.promise; jest.advanceTimersByTime(CODEX_WS_RESPONSE_PRELUDE_TIMEOUT_MS); @@ -1541,10 +1541,10 @@ describe("codexWsUpstreamFetch", () => { // The 90 s bound is a silence bound, not a deadline: a peer that answers pings is alive, // and how long an alive origin may take before response.created belongs to the client's // own deadline and the operator's connectTimeoutMs, not to a fixed number in the proxy. - const noResend = (counter: { http: number }) => (async () => { + const noResend = (counter: { http: number }) => Object.assign(async () => { counter.http++; return new Response("must not resend"); - }) as typeof fetch; + }, { preconnect: RealFetch.preconnect }); test("a peer that answers pings stays alive past the silence bound and still completes with one send", async () => { jest.useFakeTimers(); @@ -1967,7 +1967,7 @@ describe("oversized Codex create frames", () => { const response = await codexWsUpstreamFetch( "https://sub2api.example.com/v1/responses", streamingInit(), - (async () => sentinel) as typeof fetch, + Object.assign(async () => sentinel, { preconnect: RealFetch.preconnect }), ); expect(FakeWebSocket.instances).toHaveLength(0); expect(response).toBe(sentinel); diff --git a/tests/usage/quota-pool.test.ts b/tests/usage/quota-pool.test.ts index 6f613e7f271..d30e9342936 100644 --- a/tests/usage/quota-pool.test.ts +++ b/tests/usage/quota-pool.test.ts @@ -40,6 +40,7 @@ beforeAll(async () => { function config(): OcxConfig { return { + port: 0, defaultProvider: "quiet", providers: { xai: { adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "oauth" }, "claude-cli": { adapter: "claude-cli", baseUrl: "http://127.0.0.1:9" }, From 95b621d533178a1e96363ab0c9aa3c0629da070a Mon Sep 17 00:00:00 2001 From: JUN <243035832+lidge-jun@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:25:01 +0900 Subject: [PATCH 6/6] fix(codex): retain the latest applicable credit retry hold --- .../docs/reference/codex-retry-diagnostics.md | 6 ++++-- src/codex/auth-context.ts | 5 ++--- src/codex/pool-credit-policy.ts | 13 +++++++++++++ structure/providers/openai-tiers.md | 8 ++------ .../codex-credit-scoped-retry.test.ts | 9 +++++++++ 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md index bc12792d2a2..0a7112cab84 100644 --- a/docs-site/src/content/docs/reference/codex-retry-diagnostics.md +++ b/docs-site/src/content/docs/reference/codex-retry-diagnostics.md @@ -54,7 +54,8 @@ An explicitly selected stored account can reach its included usage limit while i credit balance observation is older than five minutes. If credit spending is enabled for that account and the evidence is stale, missing or incomplete, its local 429 advises retrying after the 300-second base metadata cadence (or at the included reset if sooner). A later known upstream -cooldown for the selected quota scope, including account-wide holds, still takes precedence. Unrelated +cooldown for the selected quota scope, including account-wide holds, still takes precedence; when +both apply, the later deadline wins. Unrelated model-group holds are not imported. The original included-window reset remains attached to the internal refusal as `resetAt`; the shorter hint is a metadata recheck opportunity, not a new quota reset. @@ -65,7 +66,8 @@ credit opt-out, an older cached null without provenance, a known spending veto o balance without unlimited entitlement retain the included-reset advice. A newly observed missing or malformed credit summary retracts spending authority but permits the short metadata retry; an explicit spending veto still wins. Unlimited credits with a zero balance require fresh evidence -just like other credits. Ordinary upstream 429 delays are not capped by this policy. +just like other credits. Ordinary upstream 429 delays keep the existing parser ceiling of 24 hours +for numeric and HTTP-date Retry-After values. ## Triage: `idle timeout waiting for SSE` diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index f8ba620d6b2..d6a52b96ae3 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -1,4 +1,4 @@ -import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt, poolCreditRecoveryNeeded, poolContextCreditRecoveryNeeded } from "./pool-credit-policy"; +import { bindPoolCreditPolicy, poolCreditHoldResetAt, poolContextCreditHoldResetAt, poolCreditRecoveryNeeded, poolContextCreditRecoveryNeeded, poolCreditRecoveryRetryAt } from "./pool-credit-policy"; import { hasSpendableCodexCredits } from "./quota-types"; import { nextQuotaQueryDelay } from "./quota-query-backoff"; import { noteMainAccountActivity } from "./main-account-external-usage"; @@ -550,8 +550,7 @@ export class CodexPoolAccountCreditsOffError extends CodexAccountCooldownError { constructor(accountId: string, resetAt: number, creditRecovery = false, quotaScope?: CodexQuotaScope) { // Keep the included reset; a metadata hint neither admits nor clears an upstream hold. - super(accountId, creditRecovery ? Math.max(Math.min(resetAt, Date.now() + nextQuotaQueryDelay()), - getCodexQuotaHealthSnapshot(accountId, quotaScope)?.cooldownUntil ?? 0) : resetAt); + super(accountId, creditRecovery ? poolCreditRecoveryRetryAt(accountId, resetAt, quotaScope) : resetAt); this.name = "CodexPoolAccountCreditsOffError"; this.resetAt = resetAt; this.message = creditRecovery diff --git a/src/codex/pool-credit-policy.ts b/src/codex/pool-credit-policy.ts index bec446f7e3f..e67898626f3 100644 --- a/src/codex/pool-credit-policy.ts +++ b/src/codex/pool-credit-policy.ts @@ -4,6 +4,8 @@ import { MAIN_CODEX_ACCOUNT_ID } from "./main-account"; import { isSelectableCodexPoolAccount } from "./account-id"; import { getAccountQuota } from "./quota"; import { hasSpendableCodexCredits } from "./quota-types"; +import { getCodexAccountCooldownUntil, scopedHealthFor, type CodexQuotaScope } from "./routing/health-store"; +import { nextQuotaQueryDelay } from "./quota-query-backoff"; export type PoolCreditPolicyConfig = Readonly>; type PoolContext = { kind: "pool"; accountId: string }; @@ -86,3 +88,14 @@ export function poolContextCreditRecoveryNeeded(context: PoolContext, override?: const policy = override ?? policies.get(context)?.policy; return !!policy && poolCreditRecoveryNeeded(policy, context.accountId); } + +/** Retry advice observes both applicable holds without changing routing's health precedence. */ +export function poolCreditRecoveryRetryAt( + accountId: string, resetAt: number, quotaScope?: CodexQuotaScope, now = Date.now(), +): number { + const scopedUntil = quotaScope ? scopedHealthFor(accountId, quotaScope)?.cooldownUntil : undefined; + const liveScopedUntil = typeof scopedUntil === "number" && Number.isFinite(scopedUntil) && scopedUntil > now + ? scopedUntil : 0; + return Math.max(Math.min(resetAt, now + nextQuotaQueryDelay()), + getCodexAccountCooldownUntil(accountId, now) ?? 0, liveScopedUntil); +} diff --git a/structure/providers/openai-tiers.md b/structure/providers/openai-tiers.md index 8bad9cab2fd..b32d4db3a1f 100644 --- a/structure/providers/openai-tiers.md +++ b/structure/providers/openai-tiers.md @@ -316,13 +316,9 @@ payloads cannot clear cooldowns, actual request refusals still drive cooldown/fa default-on main-account hard lock retains its separate local admission policy. Registration warmup remains conservative and does not spend paid credits to validate an exhausted account. -Stored-account refusals in `src/codex/auth-context.ts` preserve the included-window `resetAt`. -For opted-in accounts with stale, missing or incomplete credits, `src/codex/pool-credit-policy.ts` supplies a retry hint at the earlier of that reset and the 300-second base cadence from `src/codex/quota-query-backoff.ts`; the selected quota scope's later upstream cooldown takes precedence, without importing unrelated model-group holds. -HTTP remains 429 and admission stays closed. Formatting never triggers a query, clears cooldown or switches accounts; query pacing remains independent. -Opt-out, legacy null, known spending/overage vetoes and finite nonpositive balances without unlimited entitlement retain reset-bound advice as evidence ages. Parser-proven unavailable null carries `creditObservationUnavailable: true` through merges and persistence; replacement clears that provenance. It changes retry advice only and never authorizes spending. -The same live-policy classification applies after token preparation, at materialization/dispatch and through `src/providers/openai-sidecar-credit.ts`. Main policy and ordinary upstream 429 formatting are unchanged. +Stored-account refusals in `src/codex/auth-context.ts` preserve the included-window `resetAt`. For opted-in accounts with stale, missing or incomplete credits, `src/codex/pool-credit-policy.ts` supplies a retry hint at the earlier of that reset and the 300-second base cadence from `src/codex/quota-query-backoff.ts`; the latest live account-wide or selected-scope cooldown takes precedence, without importing unrelated model-group holds. Numeric and HTTP-date Retry-After input retains the existing 24-hour parser ceiling. +HTTP remains 429 and admission stays closed. Formatting never triggers a query, clears cooldown or switches accounts; query pacing remains independent. Opt-out, legacy null, known spending/overage vetoes and finite nonpositive balances without unlimited entitlement retain reset-bound advice as evidence ages. Parser-proven unavailable null carries `creditObservationUnavailable: true` through merges and persistence; replacement clears that provenance. It changes retry advice only and never authorizes spending. The same live-policy classification applies after token preparation, at materialization/dispatch and through `src/providers/openai-sidecar-credit.ts`. Main policy and ordinary upstream 429 formatting are unchanged. Coverage: `tests/codex-integration/codex-credit-retry-after.test.ts` and `tests/codex-integration/codex-credit-scoped-retry.test.ts`. - For an opted-in, unpaused main account with a currently full usage window, the existing `src/codex/auth-api/pool-mode-gate.ts` recovery sweep renews credit observations from three minutes of age, leaving time for token preparation before the five-minute freshness limit. Missing, zero, diff --git a/tests/codex-integration/codex-credit-scoped-retry.test.ts b/tests/codex-integration/codex-credit-scoped-retry.test.ts index 1946f3e3005..7c44f9a2713 100644 --- a/tests/codex-integration/codex-credit-scoped-retry.test.ts +++ b/tests/codex-integration/codex-credit-scoped-retry.test.ts @@ -80,6 +80,10 @@ const cases = [ { name: "independent Reserve hold", modelId: "gpt-reserve", retry: 300 }, { name: "account-wide Retry-After", modelId: "gpt-reserve", retryAfter: "3600", retry: 3600 }, { name: "expired shared hold", modelId: MODEL, age: 901_000, retry: 300 }, + { name: "shared hold recorded before a shorter account hold", modelId: MODEL, retryAfter: "600", combined: "before", retry: 900 }, + { name: "shared hold recorded after a shorter account hold", modelId: MODEL, retryAfter: "600", combined: "after", retry: 900 }, + { name: "oversized numeric Retry-After", modelId: MODEL, retryAfter: "99999999999999999999", retry: 86_400 }, + { name: "far-future Retry-After date", modelId: MODEL, retryAfter: "Fri, 31 Dec 9999 23:59:59 GMT", retry: 86_400 }, ] as const; for (const boundary of ["resolve", "materialize", "materialize-async", "dispatch", "provider", "cooldown", "sidecar"] as const) { for (const scenario of cases) test(`${boundary} credit refusal respects ${scenario.name}`, async () => { @@ -89,9 +93,14 @@ for (const boundary of ["resolve", "materialize", "materialize-async", "dispatch if (context) expect(context).toMatchObject({ kind: "pool", accountId: ID, quotaScope: "shared" }); observe(360_000); const recordedAt = NOW - ("age" in scenario ? scenario.age : 0); + const recordSharedHold = () => recordCodexUpstreamOutcome(config, ID, 429, { + modelId: MODEL, resetAt: NOW + 3_600_000, fixedAccount: true, now: NOW, + }); + if ("combined" in scenario && scenario.combined === "before") recordSharedHold(); recordCodexUpstreamOutcome(config, ID, 429, { modelId: scenario.modelId, resetAt: recordedAt + 3_600_000, fixedAccount: true, now: recordedAt, ...("retryAfter" in scenario ? { retryAfter: scenario.retryAfter } : {}) }); + if ("combined" in scenario && scenario.combined === "after") recordSharedHold(); const before = state(); let error: unknown; try {